303afd0339
Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-conciseness-s42 Source: Original Platform
3824 lines
143 KiB
JSON
3824 lines
143 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8594449418084154,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004774694121157864,
|
|
"grad_norm": 60.5,
|
|
"ht_mnpo/logit": 0.2415936291217804,
|
|
"ht_mnpo/residual": 0.2340935915708542,
|
|
"ht_mnpo/residual_abs": 0.23548129200935364,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -2.011902332305908,
|
|
"logits/rejected": -2.257323741912842,
|
|
"logps/chosen": -0.25824588537216187,
|
|
"logps/rejected": -0.2757476568222046,
|
|
"loss": 0.5226427316665649,
|
|
"loss/core": 0.5226427316665649,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.5988030433654785,
|
|
"rewards/margins": 2.41593599319458,
|
|
"rewards/rejected": 1.1828668117523193,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009549388242315727,
|
|
"grad_norm": 2736.0,
|
|
"ht_mnpo/logit": 0.26693910360336304,
|
|
"ht_mnpo/residual": 0.2594391107559204,
|
|
"ht_mnpo/residual_abs": 0.2632564902305603,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.0542407035827637,
|
|
"logits/rejected": -2.270714282989502,
|
|
"logps/chosen": -0.2949259281158447,
|
|
"logps/rejected": -0.29275721311569214,
|
|
"loss": 1.0850975513458252,
|
|
"loss/core": 1.0850975513458252,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.9222195148468018,
|
|
"rewards/margins": 2.669390916824341,
|
|
"rewards/rejected": 1.2528284788131714,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.01432408236347359,
|
|
"grad_norm": 10.8125,
|
|
"ht_mnpo/logit": 0.19589567184448242,
|
|
"ht_mnpo/residual": 0.188395693898201,
|
|
"ht_mnpo/residual_abs": 0.19078858196735382,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.067415475845337,
|
|
"logits/rejected": -2.327845811843872,
|
|
"logps/chosen": -0.314541757106781,
|
|
"logps/rejected": -0.29293060302734375,
|
|
"loss": 0.5424398183822632,
|
|
"loss/core": 0.5424398183822632,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4876604080200195,
|
|
"rewards/margins": 1.9589569568634033,
|
|
"rewards/rejected": 0.5287034511566162,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.019098776484631454,
|
|
"grad_norm": 36.0,
|
|
"ht_mnpo/logit": 0.3147750794887543,
|
|
"ht_mnpo/residual": 0.30727505683898926,
|
|
"ht_mnpo/residual_abs": 0.30989599227905273,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -1.927666425704956,
|
|
"logits/rejected": -2.2836575508117676,
|
|
"logps/chosen": -0.3032557964324951,
|
|
"logps/rejected": -0.2924278676509857,
|
|
"loss": 1.1851537227630615,
|
|
"loss/core": 1.1851537227630615,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.1310954093933105,
|
|
"rewards/margins": 3.1477506160736084,
|
|
"rewards/rejected": 0.9833448529243469,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.023873470605789315,
|
|
"grad_norm": 73.5,
|
|
"ht_mnpo/logit": 0.12462173402309418,
|
|
"ht_mnpo/residual": 0.11712173372507095,
|
|
"ht_mnpo/residual_abs": 0.12095322459936142,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.0298550128936768,
|
|
"logits/rejected": -2.2757391929626465,
|
|
"logps/chosen": -0.31602349877357483,
|
|
"logps/rejected": -0.3301704525947571,
|
|
"loss": 0.18879806995391846,
|
|
"loss/core": 0.18879806995391846,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6169111728668213,
|
|
"rewards/margins": 1.2462173700332642,
|
|
"rewards/rejected": 0.3706938624382019,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.02864816472694718,
|
|
"grad_norm": 368.0,
|
|
"ht_mnpo/logit": 0.12316179275512695,
|
|
"ht_mnpo/residual": 0.11566178500652313,
|
|
"ht_mnpo/residual_abs": 0.11892789602279663,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.042910575866699,
|
|
"logits/rejected": -2.2183661460876465,
|
|
"logps/chosen": -0.28492701053619385,
|
|
"logps/rejected": -0.3071616291999817,
|
|
"loss": 0.1271527111530304,
|
|
"loss/core": 0.1271527111530304,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4336202144622803,
|
|
"rewards/margins": 1.23161780834198,
|
|
"rewards/rejected": 1.2020022869110107,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033422858848105044,
|
|
"grad_norm": 9.625,
|
|
"ht_mnpo/logit": 0.3371656835079193,
|
|
"ht_mnpo/residual": 0.3296656310558319,
|
|
"ht_mnpo/residual_abs": 0.3311299681663513,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -1.9528906345367432,
|
|
"logits/rejected": -2.2516660690307617,
|
|
"logps/chosen": -0.27027541399002075,
|
|
"logps/rejected": -0.27153652906417847,
|
|
"loss": 1.1185133457183838,
|
|
"loss/core": 1.1185133457183838,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.334385395050049,
|
|
"rewards/margins": 3.3716564178466797,
|
|
"rewards/rejected": 0.9627290964126587,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03819755296926291,
|
|
"grad_norm": 14.1875,
|
|
"ht_mnpo/logit": 0.21347995102405548,
|
|
"ht_mnpo/residual": 0.20597994327545166,
|
|
"ht_mnpo/residual_abs": 0.20741498470306396,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.00959849357605,
|
|
"logits/rejected": -2.2939441204071045,
|
|
"logps/chosen": -0.26474690437316895,
|
|
"logps/rejected": -0.276932954788208,
|
|
"loss": 0.41715115308761597,
|
|
"loss/core": 0.41715115308761597,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.2156689167022705,
|
|
"rewards/margins": 2.1347994804382324,
|
|
"rewards/rejected": 1.080869436264038,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04297224709042077,
|
|
"grad_norm": 2.203125,
|
|
"ht_mnpo/logit": 0.2520568370819092,
|
|
"ht_mnpo/residual": 0.24455685913562775,
|
|
"ht_mnpo/residual_abs": 0.24652080237865448,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -2.1033449172973633,
|
|
"logits/rejected": -2.3409695625305176,
|
|
"logps/chosen": -0.2933202385902405,
|
|
"logps/rejected": -0.3012697398662567,
|
|
"loss": 0.9052979350090027,
|
|
"loss/core": 0.9052979350090027,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.102382183074951,
|
|
"rewards/margins": 2.520568609237671,
|
|
"rewards/rejected": 0.5818136930465698,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04774694121157863,
|
|
"grad_norm": 103.5,
|
|
"ht_mnpo/logit": 0.2532837688922882,
|
|
"ht_mnpo/residual": 0.24578377604484558,
|
|
"ht_mnpo/residual_abs": 0.2527261972427368,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -1.9192997217178345,
|
|
"logits/rejected": -2.134883403778076,
|
|
"logps/chosen": -0.26225295662879944,
|
|
"logps/rejected": -0.29307669401168823,
|
|
"loss": 0.6835693717002869,
|
|
"loss/core": 0.6835693717002869,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.6984927654266357,
|
|
"rewards/margins": 2.5328376293182373,
|
|
"rewards/rejected": 1.1656550168991089,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.052521635332736495,
|
|
"grad_norm": 2.328125,
|
|
"ht_mnpo/logit": 0.203359916806221,
|
|
"ht_mnpo/residual": 0.1958599090576172,
|
|
"ht_mnpo/residual_abs": 0.1985529363155365,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -1.9192931652069092,
|
|
"logits/rejected": -2.2161107063293457,
|
|
"logps/chosen": -0.2721022963523865,
|
|
"logps/rejected": -0.2915636897087097,
|
|
"loss": 0.437203973531723,
|
|
"loss/core": 0.437203973531723,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0200881958007812,
|
|
"rewards/margins": 2.0335988998413086,
|
|
"rewards/rejected": 0.9864892959594727,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.05729632945389436,
|
|
"grad_norm": 19.5,
|
|
"ht_mnpo/logit": 0.2052539587020874,
|
|
"ht_mnpo/residual": 0.19775396585464478,
|
|
"ht_mnpo/residual_abs": 0.20126383006572723,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -1.9262555837631226,
|
|
"logits/rejected": -2.166170120239258,
|
|
"logps/chosen": -0.29561707377433777,
|
|
"logps/rejected": -0.31034865975379944,
|
|
"loss": 1.1692918539047241,
|
|
"loss/core": 1.1692918539047241,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.606314182281494,
|
|
"rewards/margins": 2.052539348602295,
|
|
"rewards/rejected": 0.5537749528884888,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.062071023575052224,
|
|
"grad_norm": 924.0,
|
|
"ht_mnpo/logit": 0.2645673453807831,
|
|
"ht_mnpo/residual": 0.25706732273101807,
|
|
"ht_mnpo/residual_abs": 0.2587380111217499,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -2.020003080368042,
|
|
"logits/rejected": -2.3714873790740967,
|
|
"logps/chosen": -0.2805558741092682,
|
|
"logps/rejected": -0.27380621433258057,
|
|
"loss": 1.0117337703704834,
|
|
"loss/core": 1.0117337703704834,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.615847110748291,
|
|
"rewards/margins": 2.6456735134124756,
|
|
"rewards/rejected": 0.9701736569404602,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06684571769621009,
|
|
"grad_norm": 1.40625,
|
|
"ht_mnpo/logit": 0.14507603645324707,
|
|
"ht_mnpo/residual": 0.13757602870464325,
|
|
"ht_mnpo/residual_abs": 0.13970807194709778,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.9647560119628906,
|
|
"logits/rejected": -2.2312729358673096,
|
|
"logps/chosen": -0.28448420763015747,
|
|
"logps/rejected": -0.29094865918159485,
|
|
"loss": 0.08910230547189713,
|
|
"loss/core": 0.08910230547189713,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4837958812713623,
|
|
"rewards/margins": 1.4507603645324707,
|
|
"rewards/rejected": 1.0330355167388916,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07162041181736795,
|
|
"grad_norm": 99.5,
|
|
"ht_mnpo/logit": 0.34166499972343445,
|
|
"ht_mnpo/residual": 0.3341650068759918,
|
|
"ht_mnpo/residual_abs": 0.3365333080291748,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -1.8882007598876953,
|
|
"logits/rejected": -2.146094560623169,
|
|
"logps/chosen": -0.28841090202331543,
|
|
"logps/rejected": -0.28859004378318787,
|
|
"loss": 0.9650213122367859,
|
|
"loss/core": 0.9650213122367859,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.434047698974609,
|
|
"rewards/margins": 3.4166500568389893,
|
|
"rewards/rejected": 1.0173976421356201,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07639510593852582,
|
|
"grad_norm": 98.0,
|
|
"ht_mnpo/logit": 0.4781981110572815,
|
|
"ht_mnpo/residual": 0.47069811820983887,
|
|
"ht_mnpo/residual_abs": 0.47250255942344666,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.0093605518341064,
|
|
"logits/rejected": -2.3259353637695312,
|
|
"logps/chosen": -0.2699472904205322,
|
|
"logps/rejected": -0.2692564129829407,
|
|
"loss": 2.545933485031128,
|
|
"loss/core": 2.545933485031128,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 5.524721622467041,
|
|
"rewards/margins": 4.781980991363525,
|
|
"rewards/rejected": 0.7427404522895813,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08116980005968367,
|
|
"grad_norm": 28.375,
|
|
"ht_mnpo/logit": 0.3949056565761566,
|
|
"ht_mnpo/residual": 0.3874056935310364,
|
|
"ht_mnpo/residual_abs": 0.38943880796432495,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -1.8436777591705322,
|
|
"logits/rejected": -2.107656478881836,
|
|
"logps/chosen": -0.27149876952171326,
|
|
"logps/rejected": -0.26327431201934814,
|
|
"loss": 1.4980366230010986,
|
|
"loss/core": 1.4980366230010986,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 5.046863555908203,
|
|
"rewards/margins": 3.949056625366211,
|
|
"rewards/rejected": 1.097806692123413,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08594449418084155,
|
|
"grad_norm": 9.6875,
|
|
"ht_mnpo/logit": 0.2532029151916504,
|
|
"ht_mnpo/residual": 0.24570293724536896,
|
|
"ht_mnpo/residual_abs": 0.24952733516693115,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -1.9614741802215576,
|
|
"logits/rejected": -2.207104444503784,
|
|
"logps/chosen": -0.308493047952652,
|
|
"logps/rejected": -0.2764720320701599,
|
|
"loss": 0.856387734413147,
|
|
"loss/core": 0.856387734413147,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3289856910705566,
|
|
"rewards/margins": 2.532029151916504,
|
|
"rewards/rejected": 0.7969565391540527,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.0907191883019994,
|
|
"grad_norm": 504.0,
|
|
"ht_mnpo/logit": 0.2930489778518677,
|
|
"ht_mnpo/residual": 0.28554895520210266,
|
|
"ht_mnpo/residual_abs": 0.2872154712677002,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.090327262878418,
|
|
"logits/rejected": -2.431143283843994,
|
|
"logps/chosen": -0.2864113748073578,
|
|
"logps/rejected": -0.28018051385879517,
|
|
"loss": 1.093916654586792,
|
|
"loss/core": 1.093916654586792,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.887937545776367,
|
|
"rewards/margins": 2.9304897785186768,
|
|
"rewards/rejected": 0.9574478268623352,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09549388242315726,
|
|
"grad_norm": 43.75,
|
|
"ht_mnpo/logit": 0.13229675590991974,
|
|
"ht_mnpo/residual": 0.12479674816131592,
|
|
"ht_mnpo/residual_abs": 0.15026070177555084,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -1.9240186214447021,
|
|
"logits/rejected": -2.271806001663208,
|
|
"logps/chosen": -0.2857854664325714,
|
|
"logps/rejected": -0.3030196726322174,
|
|
"loss": 0.26169103384017944,
|
|
"loss/core": 0.26169103384017944,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.2315335273742676,
|
|
"rewards/margins": 1.322967529296875,
|
|
"rewards/rejected": 0.908565878868103,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10026857654431513,
|
|
"grad_norm": 788.0,
|
|
"ht_mnpo/logit": 0.2821234464645386,
|
|
"ht_mnpo/residual": 0.27462345361709595,
|
|
"ht_mnpo/residual_abs": 0.275934636592865,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.103553533554077,
|
|
"logits/rejected": -2.360391855239868,
|
|
"logps/chosen": -0.32235783338546753,
|
|
"logps/rejected": -0.29607582092285156,
|
|
"loss": 0.564530611038208,
|
|
"loss/core": 0.564530611038208,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 4.4333977699279785,
|
|
"rewards/margins": 2.8212342262268066,
|
|
"rewards/rejected": 1.6121633052825928,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10504327066547299,
|
|
"grad_norm": 122.0,
|
|
"ht_mnpo/logit": 0.29163163900375366,
|
|
"ht_mnpo/residual": 0.28413161635398865,
|
|
"ht_mnpo/residual_abs": 0.28595611453056335,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -2.0258805751800537,
|
|
"logits/rejected": -2.2438912391662598,
|
|
"logps/chosen": -0.289029598236084,
|
|
"logps/rejected": -0.2787890136241913,
|
|
"loss": 1.244662880897522,
|
|
"loss/core": 1.244662880897522,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.081713676452637,
|
|
"rewards/margins": 2.916316032409668,
|
|
"rewards/rejected": 1.1653969287872314,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10981796478663086,
|
|
"grad_norm": 58.5,
|
|
"ht_mnpo/logit": 0.2460227906703949,
|
|
"ht_mnpo/residual": 0.23852276802062988,
|
|
"ht_mnpo/residual_abs": 0.24091574549674988,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -1.842900276184082,
|
|
"logits/rejected": -2.1219844818115234,
|
|
"logps/chosen": -0.27585095167160034,
|
|
"logps/rejected": -0.26493367552757263,
|
|
"loss": 1.031520128250122,
|
|
"loss/core": 1.031520128250122,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.6426234245300293,
|
|
"rewards/margins": 2.4602277278900146,
|
|
"rewards/rejected": 1.1823960542678833,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11459265890778872,
|
|
"grad_norm": 516.0,
|
|
"ht_mnpo/logit": 0.33325010538101196,
|
|
"ht_mnpo/residual": 0.32575008273124695,
|
|
"ht_mnpo/residual_abs": 0.3277893662452698,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -2.084425687789917,
|
|
"logits/rejected": -2.2928617000579834,
|
|
"logps/chosen": -0.2503313422203064,
|
|
"logps/rejected": -0.27189603447914124,
|
|
"loss": 0.949650764465332,
|
|
"loss/core": 0.949650764465332,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.397518157958984,
|
|
"rewards/margins": 3.332500457763672,
|
|
"rewards/rejected": 1.0650172233581543,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11936735302894658,
|
|
"grad_norm": 756.0,
|
|
"ht_mnpo/logit": 0.4765322208404541,
|
|
"ht_mnpo/residual": 0.4690321981906891,
|
|
"ht_mnpo/residual_abs": 0.47166210412979126,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -1.8222243785858154,
|
|
"logits/rejected": -2.102375030517578,
|
|
"logps/chosen": -0.2784605026245117,
|
|
"logps/rejected": -0.2793668210506439,
|
|
"loss": 2.5388500690460205,
|
|
"loss/core": 2.5388500690460205,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 5.699808120727539,
|
|
"rewards/margins": 4.765322208404541,
|
|
"rewards/rejected": 0.9344862103462219,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.12414204715010445,
|
|
"grad_norm": 18.875,
|
|
"ht_mnpo/logit": 0.28754621744155884,
|
|
"ht_mnpo/residual": 0.2800462245941162,
|
|
"ht_mnpo/residual_abs": 0.28318971395492554,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -1.8026081323623657,
|
|
"logits/rejected": -2.1300437450408936,
|
|
"logps/chosen": -0.3127506673336029,
|
|
"logps/rejected": -0.2983434796333313,
|
|
"loss": 0.685983419418335,
|
|
"loss/core": 0.685983419418335,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.466167449951172,
|
|
"rewards/margins": 2.875462293624878,
|
|
"rewards/rejected": 0.590705156326294,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12891674127126232,
|
|
"grad_norm": 248.0,
|
|
"ht_mnpo/logit": 0.18734104931354523,
|
|
"ht_mnpo/residual": 0.1798410415649414,
|
|
"ht_mnpo/residual_abs": 0.18434356153011322,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.0444672107696533,
|
|
"logits/rejected": -2.295093059539795,
|
|
"logps/chosen": -0.28641682863235474,
|
|
"logps/rejected": -0.29985663294792175,
|
|
"loss": 0.27129822969436646,
|
|
"loss/core": 0.27129822969436646,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.71679949760437,
|
|
"rewards/margins": 1.8734104633331299,
|
|
"rewards/rejected": 0.8433891534805298,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13369143539242018,
|
|
"grad_norm": 2.171875,
|
|
"ht_mnpo/logit": 0.1653221845626831,
|
|
"ht_mnpo/residual": 0.15782217681407928,
|
|
"ht_mnpo/residual_abs": 0.16011430323123932,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -1.931020736694336,
|
|
"logits/rejected": -2.2295100688934326,
|
|
"logps/chosen": -0.28395944833755493,
|
|
"logps/rejected": -0.27869370579719543,
|
|
"loss": 0.12267724424600601,
|
|
"loss/core": 0.12267724424600601,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6366994380950928,
|
|
"rewards/margins": 1.6532217264175415,
|
|
"rewards/rejected": 0.9834780693054199,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13846612951357803,
|
|
"grad_norm": 0.75,
|
|
"ht_mnpo/logit": 0.2931879162788391,
|
|
"ht_mnpo/residual": 0.28568798303604126,
|
|
"ht_mnpo/residual_abs": 0.2914429306983948,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.8905494213104248,
|
|
"logits/rejected": -2.122390031814575,
|
|
"logps/chosen": -0.3317320942878723,
|
|
"logps/rejected": -0.30477237701416016,
|
|
"loss": 1.277919054031372,
|
|
"loss/core": 1.277919054031372,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.9621856212615967,
|
|
"rewards/margins": 2.9318792819976807,
|
|
"rewards/rejected": 1.0303064584732056,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.1432408236347359,
|
|
"grad_norm": 1.546875,
|
|
"ht_mnpo/logit": 0.2134993076324463,
|
|
"ht_mnpo/residual": 0.20599932968616486,
|
|
"ht_mnpo/residual_abs": 0.20773494243621826,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.0588443279266357,
|
|
"logits/rejected": -2.308776378631592,
|
|
"logps/chosen": -0.2723400592803955,
|
|
"logps/rejected": -0.2848970293998718,
|
|
"loss": 0.3919183015823364,
|
|
"loss/core": 0.3919183015823364,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.260389804840088,
|
|
"rewards/margins": 2.134993076324463,
|
|
"rewards/rejected": 1.1253963708877563,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.14801551775589375,
|
|
"grad_norm": 165.0,
|
|
"ht_mnpo/logit": 0.32777029275894165,
|
|
"ht_mnpo/residual": 0.32027024030685425,
|
|
"ht_mnpo/residual_abs": 0.3234081268310547,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.0180845260620117,
|
|
"logits/rejected": -2.2344937324523926,
|
|
"logps/chosen": -0.3052632808685303,
|
|
"logps/rejected": -0.2794122099876404,
|
|
"loss": 1.312514066696167,
|
|
"loss/core": 1.312514066696167,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.1154704093933105,
|
|
"rewards/margins": 3.277703046798706,
|
|
"rewards/rejected": 0.8377677798271179,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15279021187705163,
|
|
"grad_norm": 89.5,
|
|
"ht_mnpo/logit": 0.15068449079990387,
|
|
"ht_mnpo/residual": 0.14318448305130005,
|
|
"ht_mnpo/residual_abs": 0.14576324820518494,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.039623975753784,
|
|
"logits/rejected": -2.4383645057678223,
|
|
"logps/chosen": -0.26783841848373413,
|
|
"logps/rejected": -0.27481716871261597,
|
|
"loss": 0.16402561962604523,
|
|
"loss/core": 0.16402561962604523,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1534738540649414,
|
|
"rewards/margins": 1.5068447589874268,
|
|
"rewards/rejected": 0.6466289758682251,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.1575649059982095,
|
|
"grad_norm": 3.75,
|
|
"ht_mnpo/logit": 0.37046578526496887,
|
|
"ht_mnpo/residual": 0.36296579241752625,
|
|
"ht_mnpo/residual_abs": 0.36714157462120056,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.004047393798828,
|
|
"logits/rejected": -2.2582778930664062,
|
|
"logps/chosen": -0.2819061875343323,
|
|
"logps/rejected": -0.2967601418495178,
|
|
"loss": 1.949097990989685,
|
|
"loss/core": 1.949097990989685,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.285637855529785,
|
|
"rewards/margins": 3.704658031463623,
|
|
"rewards/rejected": 0.5809799432754517,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16233960011936735,
|
|
"grad_norm": 12.625,
|
|
"ht_mnpo/logit": 0.2548084855079651,
|
|
"ht_mnpo/residual": 0.24730846285820007,
|
|
"ht_mnpo/residual_abs": 0.24948911368846893,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.0560150146484375,
|
|
"logits/rejected": -2.2419166564941406,
|
|
"logps/chosen": -0.3087208867073059,
|
|
"logps/rejected": -0.32812321186065674,
|
|
"loss": 0.8445270657539368,
|
|
"loss/core": 0.8445270657539368,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.7604167461395264,
|
|
"rewards/margins": 2.5480847358703613,
|
|
"rewards/rejected": 1.212332010269165,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.1671142942405252,
|
|
"grad_norm": 25.25,
|
|
"ht_mnpo/logit": 0.20657019317150116,
|
|
"ht_mnpo/residual": 0.19907021522521973,
|
|
"ht_mnpo/residual_abs": 0.20315143465995789,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.158867597579956,
|
|
"logits/rejected": -2.4179515838623047,
|
|
"logps/chosen": -0.29173943400382996,
|
|
"logps/rejected": -0.3005591630935669,
|
|
"loss": 1.0301963090896606,
|
|
"loss/core": 1.0301963090896606,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7829740047454834,
|
|
"rewards/margins": 2.065701961517334,
|
|
"rewards/rejected": 0.7172719240188599,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.1718889883616831,
|
|
"grad_norm": 2.28125,
|
|
"ht_mnpo/logit": 0.25792625546455383,
|
|
"ht_mnpo/residual": 0.2504262328147888,
|
|
"ht_mnpo/residual_abs": 0.2523646354675293,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -1.9154199361801147,
|
|
"logits/rejected": -2.108656644821167,
|
|
"logps/chosen": -0.29020223021507263,
|
|
"logps/rejected": -0.28535544872283936,
|
|
"loss": 0.9428588151931763,
|
|
"loss/core": 0.9428588151931763,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.405637741088867,
|
|
"rewards/margins": 2.5792624950408936,
|
|
"rewards/rejected": 0.8263753056526184,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17666368248284095,
|
|
"grad_norm": 9.1875,
|
|
"ht_mnpo/logit": 0.11725939810276031,
|
|
"ht_mnpo/residual": 0.1097593903541565,
|
|
"ht_mnpo/residual_abs": 0.1118379607796669,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -2.0307536125183105,
|
|
"logits/rejected": -2.279958486557007,
|
|
"logps/chosen": -0.2843839228153229,
|
|
"logps/rejected": -0.2895074784755707,
|
|
"loss": 0.05697331950068474,
|
|
"loss/core": 0.05697331950068474,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.8723161220550537,
|
|
"rewards/margins": 1.1725938320159912,
|
|
"rewards/rejected": 0.6997222900390625,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.1814383766039988,
|
|
"grad_norm": 314.0,
|
|
"ht_mnpo/logit": 0.2137770652770996,
|
|
"ht_mnpo/residual": 0.2062770575284958,
|
|
"ht_mnpo/residual_abs": 0.21894094347953796,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -1.7537132501602173,
|
|
"logits/rejected": -1.9378242492675781,
|
|
"logps/chosen": -0.29650992155075073,
|
|
"logps/rejected": -0.2991756796836853,
|
|
"loss": 0.4277607798576355,
|
|
"loss/core": 0.4277607798576355,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.179126262664795,
|
|
"rewards/margins": 2.137770652770996,
|
|
"rewards/rejected": 2.041355609893799,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18621307072515667,
|
|
"grad_norm": 616.0,
|
|
"ht_mnpo/logit": 0.28763315081596375,
|
|
"ht_mnpo/residual": 0.2801331579685211,
|
|
"ht_mnpo/residual_abs": 0.2813577353954315,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.9013550281524658,
|
|
"logits/rejected": -2.1618947982788086,
|
|
"logps/chosen": -0.2702489197254181,
|
|
"logps/rejected": -0.2578398883342743,
|
|
"loss": 0.7751284837722778,
|
|
"loss/core": 0.7751284837722778,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 4.433821678161621,
|
|
"rewards/margins": 2.876331329345703,
|
|
"rewards/rejected": 1.557490587234497,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19098776484631452,
|
|
"grad_norm": 764.0,
|
|
"ht_mnpo/logit": 0.36720559000968933,
|
|
"ht_mnpo/residual": 0.3597055971622467,
|
|
"ht_mnpo/residual_abs": 0.3611852824687958,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.9602934122085571,
|
|
"logits/rejected": -2.284992218017578,
|
|
"logps/chosen": -0.30019837617874146,
|
|
"logps/rejected": -0.29457828402519226,
|
|
"loss": 1.2417619228363037,
|
|
"loss/core": 1.2417619228363037,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 4.696887969970703,
|
|
"rewards/margins": 3.672055721282959,
|
|
"rewards/rejected": 1.024832010269165,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.1957624589674724,
|
|
"grad_norm": 1.15625,
|
|
"ht_mnpo/logit": 0.13184615969657898,
|
|
"ht_mnpo/residual": 0.12434617429971695,
|
|
"ht_mnpo/residual_abs": 0.12654730677604675,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -2.0989491939544678,
|
|
"logits/rejected": -2.2950825691223145,
|
|
"logps/chosen": -0.27947577834129333,
|
|
"logps/rejected": -0.2976261079311371,
|
|
"loss": 0.12396552413702011,
|
|
"loss/core": 0.12396552413702011,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.6087183952331543,
|
|
"rewards/margins": 1.3184616565704346,
|
|
"rewards/rejected": 1.2902568578720093,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20053715308863027,
|
|
"grad_norm": 1144.0,
|
|
"ht_mnpo/logit": 0.2897939682006836,
|
|
"ht_mnpo/residual": 0.2822939157485962,
|
|
"ht_mnpo/residual_abs": 0.28489789366722107,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -1.7683238983154297,
|
|
"logits/rejected": -2.0142149925231934,
|
|
"logps/chosen": -0.30294543504714966,
|
|
"logps/rejected": -0.29282546043395996,
|
|
"loss": 0.8169538378715515,
|
|
"loss/core": 0.8169538378715515,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.211258888244629,
|
|
"rewards/margins": 2.8979392051696777,
|
|
"rewards/rejected": 1.3133190870285034,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20531184720978812,
|
|
"grad_norm": 2.859375,
|
|
"ht_mnpo/logit": 0.19056037068367004,
|
|
"ht_mnpo/residual": 0.18306037783622742,
|
|
"ht_mnpo/residual_abs": 0.1847141683101654,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.104823589324951,
|
|
"logits/rejected": -2.3929922580718994,
|
|
"logps/chosen": -0.27915796637535095,
|
|
"logps/rejected": -0.28238147497177124,
|
|
"loss": 0.5005974769592285,
|
|
"loss/core": 0.5005974769592285,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 2.5159075260162354,
|
|
"rewards/margins": 1.9056037664413452,
|
|
"rewards/rejected": 0.610303521156311,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21008654133094598,
|
|
"grad_norm": 71.0,
|
|
"ht_mnpo/logit": 0.2221338003873825,
|
|
"ht_mnpo/residual": 0.21463382244110107,
|
|
"ht_mnpo/residual_abs": 0.21771471202373505,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -1.793410301208496,
|
|
"logits/rejected": -2.138627767562866,
|
|
"logps/chosen": -0.2802343964576721,
|
|
"logps/rejected": -0.29542267322540283,
|
|
"loss": 0.4375545084476471,
|
|
"loss/core": 0.4375545084476471,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.305910587310791,
|
|
"rewards/margins": 2.2213380336761475,
|
|
"rewards/rejected": 1.0845725536346436,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.21486123545210384,
|
|
"grad_norm": 23.5,
|
|
"ht_mnpo/logit": 0.20983533561229706,
|
|
"ht_mnpo/residual": 0.20233532786369324,
|
|
"ht_mnpo/residual_abs": 0.20403678715229034,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.118230104446411,
|
|
"logits/rejected": -2.3553214073181152,
|
|
"logps/chosen": -0.2552695572376251,
|
|
"logps/rejected": -0.27933183312416077,
|
|
"loss": 0.59180748462677,
|
|
"loss/core": 0.59180748462677,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.013653039932251,
|
|
"rewards/margins": 2.098353147506714,
|
|
"rewards/rejected": 0.9152997732162476,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21963592957326172,
|
|
"grad_norm": 4.125,
|
|
"ht_mnpo/logit": 0.13968196511268616,
|
|
"ht_mnpo/residual": 0.13218197226524353,
|
|
"ht_mnpo/residual_abs": 0.13623417913913727,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -1.9094196557998657,
|
|
"logits/rejected": -2.1887524127960205,
|
|
"logps/chosen": -0.29535284638404846,
|
|
"logps/rejected": -0.30087095499038696,
|
|
"loss": 0.09830106049776077,
|
|
"loss/core": 0.09830106049776077,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2886228561401367,
|
|
"rewards/margins": 1.3968197107315063,
|
|
"rewards/rejected": 0.8918031454086304,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.22441062369441958,
|
|
"grad_norm": 10.5625,
|
|
"ht_mnpo/logit": 0.2398790419101715,
|
|
"ht_mnpo/residual": 0.2323790341615677,
|
|
"ht_mnpo/residual_abs": 0.23418621718883514,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -1.7939506769180298,
|
|
"logits/rejected": -2.1177468299865723,
|
|
"logps/chosen": -0.2927335500717163,
|
|
"logps/rejected": -0.2929953336715698,
|
|
"loss": 0.39657431840896606,
|
|
"loss/core": 0.39657431840896606,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.5993685722351074,
|
|
"rewards/margins": 2.3987903594970703,
|
|
"rewards/rejected": 1.2005784511566162,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22918531781557744,
|
|
"grad_norm": 0.52734375,
|
|
"ht_mnpo/logit": 0.23736481368541718,
|
|
"ht_mnpo/residual": 0.22986480593681335,
|
|
"ht_mnpo/residual_abs": 0.23360922932624817,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.2269482612609863,
|
|
"logits/rejected": -2.406406879425049,
|
|
"logps/chosen": -0.30673980712890625,
|
|
"logps/rejected": -0.2728860080242157,
|
|
"loss": 0.7180436253547668,
|
|
"loss/core": 0.7180436253547668,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5076498985290527,
|
|
"rewards/margins": 2.373648166656494,
|
|
"rewards/rejected": 1.1340019702911377,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.2339600119367353,
|
|
"grad_norm": 14.9375,
|
|
"ht_mnpo/logit": 0.11371958255767822,
|
|
"ht_mnpo/residual": 0.1062195748090744,
|
|
"ht_mnpo/residual_abs": 0.1083589419722557,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.077096700668335,
|
|
"logits/rejected": -2.2862114906311035,
|
|
"logps/chosen": -0.3035627007484436,
|
|
"logps/rejected": -0.3022594153881073,
|
|
"loss": 0.10577340424060822,
|
|
"loss/core": 0.10577340424060822,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.866857886314392,
|
|
"rewards/margins": 1.1371958255767822,
|
|
"rewards/rejected": 0.7296620607376099,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23873470605789315,
|
|
"grad_norm": 44.5,
|
|
"ht_mnpo/logit": 0.12031557410955429,
|
|
"ht_mnpo/residual": 0.11281557381153107,
|
|
"ht_mnpo/residual_abs": 0.1145239844918251,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -2.05730938911438,
|
|
"logits/rejected": -2.210094928741455,
|
|
"logps/chosen": -0.2798473536968231,
|
|
"logps/rejected": -0.292803019285202,
|
|
"loss": 0.09960635006427765,
|
|
"loss/core": 0.09960635006427765,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.7082903385162354,
|
|
"rewards/margins": 1.203155755996704,
|
|
"rewards/rejected": 0.505134642124176,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24350940017905104,
|
|
"grad_norm": 4.5625,
|
|
"ht_mnpo/logit": 0.2883337438106537,
|
|
"ht_mnpo/residual": 0.28083378076553345,
|
|
"ht_mnpo/residual_abs": 0.28313925862312317,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -1.9648189544677734,
|
|
"logits/rejected": -2.2407779693603516,
|
|
"logps/chosen": -0.3144630789756775,
|
|
"logps/rejected": -0.3002500832080841,
|
|
"loss": 1.097538709640503,
|
|
"loss/core": 1.097538709640503,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.546959400177002,
|
|
"rewards/margins": 2.8833374977111816,
|
|
"rewards/rejected": 0.6636216640472412,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2482840943002089,
|
|
"grad_norm": 108.5,
|
|
"ht_mnpo/logit": 0.2983548939228058,
|
|
"ht_mnpo/residual": 0.29085490107536316,
|
|
"ht_mnpo/residual_abs": 0.2929629981517792,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -1.9292614459991455,
|
|
"logits/rejected": -2.2683722972869873,
|
|
"logps/chosen": -0.2792973816394806,
|
|
"logps/rejected": -0.310756653547287,
|
|
"loss": 1.0083863735198975,
|
|
"loss/core": 1.0083863735198975,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.546018600463867,
|
|
"rewards/margins": 2.983549118041992,
|
|
"rewards/rejected": 0.5624698400497437,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25305878842136675,
|
|
"grad_norm": 29.375,
|
|
"ht_mnpo/logit": 0.22387555241584778,
|
|
"ht_mnpo/residual": 0.21637554466724396,
|
|
"ht_mnpo/residual_abs": 0.2206393927335739,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.173464059829712,
|
|
"logits/rejected": -2.459009885787964,
|
|
"logps/chosen": -0.2751771807670593,
|
|
"logps/rejected": -0.2682764530181885,
|
|
"loss": 0.4876701831817627,
|
|
"loss/core": 0.4876701831817627,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.012019634246826,
|
|
"rewards/margins": 2.238755464553833,
|
|
"rewards/rejected": 0.7732640504837036,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.25783348254252464,
|
|
"grad_norm": 888.0,
|
|
"ht_mnpo/logit": 0.4514991343021393,
|
|
"ht_mnpo/residual": 0.44399914145469666,
|
|
"ht_mnpo/residual_abs": 0.44612884521484375,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.8616456985473633,
|
|
"logits/rejected": -2.184025764465332,
|
|
"logps/chosen": -0.2934468686580658,
|
|
"logps/rejected": -0.2970571517944336,
|
|
"loss": 2.1650872230529785,
|
|
"loss/core": 2.1650872230529785,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 5.505021572113037,
|
|
"rewards/margins": 4.514991283416748,
|
|
"rewards/rejected": 0.9900302886962891,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.26260817666368247,
|
|
"grad_norm": 2.640625,
|
|
"ht_mnpo/logit": 0.14674557745456696,
|
|
"ht_mnpo/residual": 0.13924556970596313,
|
|
"ht_mnpo/residual_abs": 0.14186374843120575,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.0834450721740723,
|
|
"logits/rejected": -2.3231890201568604,
|
|
"logps/chosen": -0.27442002296447754,
|
|
"logps/rejected": -0.27742353081703186,
|
|
"loss": 0.23988012969493866,
|
|
"loss/core": 0.23988012969493866,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.230260133743286,
|
|
"rewards/margins": 1.4674557447433472,
|
|
"rewards/rejected": 0.7628042101860046,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26738287078484035,
|
|
"grad_norm": 8.1875,
|
|
"ht_mnpo/logit": 0.2650841772556305,
|
|
"ht_mnpo/residual": 0.2575841546058655,
|
|
"ht_mnpo/residual_abs": 0.2594790458679199,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.0836455821990967,
|
|
"logits/rejected": -2.3042664527893066,
|
|
"logps/chosen": -0.32237258553504944,
|
|
"logps/rejected": -0.3145085573196411,
|
|
"loss": 1.0589872598648071,
|
|
"loss/core": 1.0589872598648071,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.507110118865967,
|
|
"rewards/margins": 2.650841474533081,
|
|
"rewards/rejected": 0.8562685251235962,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2721575649059982,
|
|
"grad_norm": 1.4765625,
|
|
"ht_mnpo/logit": 0.10140925645828247,
|
|
"ht_mnpo/residual": 0.09390926361083984,
|
|
"ht_mnpo/residual_abs": 0.09555725753307343,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -1.9129667282104492,
|
|
"logits/rejected": -2.1948471069335938,
|
|
"logps/chosen": -0.2781878709793091,
|
|
"logps/rejected": -0.2911220192909241,
|
|
"loss": 0.03205167502164841,
|
|
"loss/core": 0.03205167502164841,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 1.7170699834823608,
|
|
"rewards/margins": 1.0140926837921143,
|
|
"rewards/rejected": 0.7029772996902466,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27693225902715607,
|
|
"grad_norm": 3.875,
|
|
"ht_mnpo/logit": 0.30378761887550354,
|
|
"ht_mnpo/residual": 0.2962876260280609,
|
|
"ht_mnpo/residual_abs": 0.29867178201675415,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -1.9081875085830688,
|
|
"logits/rejected": -2.3261563777923584,
|
|
"logps/chosen": -0.2825949788093567,
|
|
"logps/rejected": -0.2706306278705597,
|
|
"loss": 1.074646234512329,
|
|
"loss/core": 1.074646234512329,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.7699265480041504,
|
|
"rewards/margins": 3.0378763675689697,
|
|
"rewards/rejected": 0.7320500016212463,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28170695314831395,
|
|
"grad_norm": 1760.0,
|
|
"ht_mnpo/logit": 0.3823835849761963,
|
|
"ht_mnpo/residual": 0.37488359212875366,
|
|
"ht_mnpo/residual_abs": 0.3766990602016449,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -1.8789278268814087,
|
|
"logits/rejected": -2.110046625137329,
|
|
"logps/chosen": -0.28207749128341675,
|
|
"logps/rejected": -0.2834545373916626,
|
|
"loss": 1.690529465675354,
|
|
"loss/core": 1.690529465675354,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.7853851318359375,
|
|
"rewards/margins": 3.823835849761963,
|
|
"rewards/rejected": 0.961548924446106,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.2864816472694718,
|
|
"grad_norm": 1.203125,
|
|
"ht_mnpo/logit": 0.1465030014514923,
|
|
"ht_mnpo/residual": 0.13900300860404968,
|
|
"ht_mnpo/residual_abs": 0.14254584908485413,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.9352973699569702,
|
|
"logits/rejected": -2.1947338581085205,
|
|
"logps/chosen": -0.32196083664894104,
|
|
"logps/rejected": -0.3307940363883972,
|
|
"loss": 0.3272203505039215,
|
|
"loss/core": 0.3272203505039215,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.2242655754089355,
|
|
"rewards/margins": 1.4650301933288574,
|
|
"rewards/rejected": 0.7592355608940125,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.29125634139062967,
|
|
"grad_norm": 99.5,
|
|
"ht_mnpo/logit": 0.13784225285053253,
|
|
"ht_mnpo/residual": 0.1303422749042511,
|
|
"ht_mnpo/residual_abs": 0.1337766945362091,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -2.080651044845581,
|
|
"logits/rejected": -2.253840684890747,
|
|
"logps/chosen": -0.30505985021591187,
|
|
"logps/rejected": -0.32858914136886597,
|
|
"loss": 0.3612667918205261,
|
|
"loss/core": 0.3612667918205261,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7895835638046265,
|
|
"rewards/margins": 1.378422498703003,
|
|
"rewards/rejected": 0.4111608862876892,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2960310355117875,
|
|
"grad_norm": 640.0,
|
|
"ht_mnpo/logit": 0.2148059904575348,
|
|
"ht_mnpo/residual": 0.20730595290660858,
|
|
"ht_mnpo/residual_abs": 0.21532678604125977,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -2.0484871864318848,
|
|
"logits/rejected": -2.299739360809326,
|
|
"logps/chosen": -0.2571372389793396,
|
|
"logps/rejected": -0.2654029428958893,
|
|
"loss": 0.6419556736946106,
|
|
"loss/core": 0.6419556736946106,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3131752014160156,
|
|
"rewards/margins": 2.148059368133545,
|
|
"rewards/rejected": 1.1651160717010498,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.3008057296329454,
|
|
"grad_norm": 6.25,
|
|
"ht_mnpo/logit": 0.26450738310813904,
|
|
"ht_mnpo/residual": 0.2570074200630188,
|
|
"ht_mnpo/residual_abs": 0.2596099078655243,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.8389809131622314,
|
|
"logits/rejected": -2.1140105724334717,
|
|
"logps/chosen": -0.289066880941391,
|
|
"logps/rejected": -0.2824665904045105,
|
|
"loss": 0.9795834422111511,
|
|
"loss/core": 0.9795834422111511,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.305142641067505,
|
|
"rewards/margins": 2.6450741291046143,
|
|
"rewards/rejected": 0.6600683331489563,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.30558042375410327,
|
|
"grad_norm": 2.671875,
|
|
"ht_mnpo/logit": 0.22446127235889435,
|
|
"ht_mnpo/residual": 0.21696126461029053,
|
|
"ht_mnpo/residual_abs": 0.2195734977722168,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -2.0431623458862305,
|
|
"logits/rejected": -2.303877830505371,
|
|
"logps/chosen": -0.2965102195739746,
|
|
"logps/rejected": -0.2953088879585266,
|
|
"loss": 0.659471333026886,
|
|
"loss/core": 0.659471333026886,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1237876415252686,
|
|
"rewards/margins": 2.244612693786621,
|
|
"rewards/rejected": 0.8791753053665161,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.3103551178752611,
|
|
"grad_norm": 528.0,
|
|
"ht_mnpo/logit": 0.4989927411079407,
|
|
"ht_mnpo/residual": 0.49149274826049805,
|
|
"ht_mnpo/residual_abs": 0.49372735619544983,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -1.8436063528060913,
|
|
"logits/rejected": -2.1103482246398926,
|
|
"logps/chosen": -0.2776651084423065,
|
|
"logps/rejected": -0.2627367377281189,
|
|
"loss": 2.5948891639709473,
|
|
"loss/core": 2.5948891639709473,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 6.395821571350098,
|
|
"rewards/margins": 4.989927768707275,
|
|
"rewards/rejected": 1.4058938026428223,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.315129811996419,
|
|
"grad_norm": 84.5,
|
|
"ht_mnpo/logit": 0.11850211769342422,
|
|
"ht_mnpo/residual": 0.1110021248459816,
|
|
"ht_mnpo/residual_abs": 0.11589847505092621,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.046609401702881,
|
|
"logits/rejected": -2.265373945236206,
|
|
"logps/chosen": -0.27779677510261536,
|
|
"logps/rejected": -0.27654415369033813,
|
|
"loss": 0.10753259807825089,
|
|
"loss/core": 0.10753259807825089,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0452699661254883,
|
|
"rewards/margins": 1.185021162033081,
|
|
"rewards/rejected": 0.8602487444877625,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.31990450611757687,
|
|
"grad_norm": 26.375,
|
|
"ht_mnpo/logit": 0.22499430179595947,
|
|
"ht_mnpo/residual": 0.21749427914619446,
|
|
"ht_mnpo/residual_abs": 0.21924762427806854,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.0356459617614746,
|
|
"logits/rejected": -2.332686424255371,
|
|
"logps/chosen": -0.26844361424446106,
|
|
"logps/rejected": -0.2597638666629791,
|
|
"loss": 0.8870676755905151,
|
|
"loss/core": 0.8870676755905151,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.095395088195801,
|
|
"rewards/margins": 2.2499425411224365,
|
|
"rewards/rejected": 0.8454524874687195,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3246792002387347,
|
|
"grad_norm": 5.34375,
|
|
"ht_mnpo/logit": 0.1895650327205658,
|
|
"ht_mnpo/residual": 0.18206503987312317,
|
|
"ht_mnpo/residual_abs": 0.18503603339195251,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -1.8680092096328735,
|
|
"logits/rejected": -2.092318296432495,
|
|
"logps/chosen": -0.3079831302165985,
|
|
"logps/rejected": -0.28415459394454956,
|
|
"loss": 0.19596347212791443,
|
|
"loss/core": 0.19596347212791443,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.789630651473999,
|
|
"rewards/margins": 1.8956505060195923,
|
|
"rewards/rejected": 0.8939803242683411,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3294538943598926,
|
|
"grad_norm": 70.0,
|
|
"ht_mnpo/logit": 0.19111791253089905,
|
|
"ht_mnpo/residual": 0.18361787497997284,
|
|
"ht_mnpo/residual_abs": 0.1863330900669098,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -2.109628915786743,
|
|
"logits/rejected": -2.291822910308838,
|
|
"logps/chosen": -0.3013684153556824,
|
|
"logps/rejected": -0.2833828628063202,
|
|
"loss": 0.2935481667518616,
|
|
"loss/core": 0.2935481667518616,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.4318795204162598,
|
|
"rewards/margins": 1.9111789464950562,
|
|
"rewards/rejected": 1.5207008123397827,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.3342285884810504,
|
|
"grad_norm": 21.375,
|
|
"ht_mnpo/logit": 0.17235636711120605,
|
|
"ht_mnpo/residual": 0.16485637426376343,
|
|
"ht_mnpo/residual_abs": 0.16569381952285767,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.0808799266815186,
|
|
"logits/rejected": -2.3465583324432373,
|
|
"logps/chosen": -0.2933531403541565,
|
|
"logps/rejected": -0.29250794649124146,
|
|
"loss": 0.18648609519004822,
|
|
"loss/core": 0.18648609519004822,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 2.39286470413208,
|
|
"rewards/margins": 1.723563551902771,
|
|
"rewards/rejected": 0.6693009734153748,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3390032826022083,
|
|
"grad_norm": 8.8125,
|
|
"ht_mnpo/logit": 0.083734892308712,
|
|
"ht_mnpo/residual": 0.07623489201068878,
|
|
"ht_mnpo/residual_abs": 0.08302940428256989,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -1.957427978515625,
|
|
"logits/rejected": -2.2546541690826416,
|
|
"logps/chosen": -0.2971518635749817,
|
|
"logps/rejected": -0.29797545075416565,
|
|
"loss": 0.02712792530655861,
|
|
"loss/core": 0.02712792530655861,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.441888451576233,
|
|
"rewards/margins": 0.8373489379882812,
|
|
"rewards/rejected": 0.6045395135879517,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.3437779767233662,
|
|
"grad_norm": 12.0,
|
|
"ht_mnpo/logit": 0.13458336889743805,
|
|
"ht_mnpo/residual": 0.12708337604999542,
|
|
"ht_mnpo/residual_abs": 0.12896767258644104,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -1.9576762914657593,
|
|
"logits/rejected": -2.2406909465789795,
|
|
"logps/chosen": -0.3130984306335449,
|
|
"logps/rejected": -0.3136448860168457,
|
|
"loss": 0.10641677677631378,
|
|
"loss/core": 0.10641677677631378,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.3418941497802734,
|
|
"rewards/margins": 1.3458335399627686,
|
|
"rewards/rejected": 0.9960603713989258,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.348552670844524,
|
|
"grad_norm": 37.25,
|
|
"ht_mnpo/logit": 0.25376901030540466,
|
|
"ht_mnpo/residual": 0.24626903235912323,
|
|
"ht_mnpo/residual_abs": 0.24840883910655975,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -1.9573538303375244,
|
|
"logits/rejected": -2.191314458847046,
|
|
"logps/chosen": -0.28709596395492554,
|
|
"logps/rejected": -0.28582268953323364,
|
|
"loss": 0.5417178273200989,
|
|
"loss/core": 0.5417178273200989,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.668320894241333,
|
|
"rewards/margins": 2.5376901626586914,
|
|
"rewards/rejected": 1.1306307315826416,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3533273649656819,
|
|
"grad_norm": 23.5,
|
|
"ht_mnpo/logit": 0.1474151760339737,
|
|
"ht_mnpo/residual": 0.13991518318653107,
|
|
"ht_mnpo/residual_abs": 0.14576002955436707,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.0737175941467285,
|
|
"logits/rejected": -2.3785977363586426,
|
|
"logps/chosen": -0.303182452917099,
|
|
"logps/rejected": -0.3058184087276459,
|
|
"loss": 0.16569259762763977,
|
|
"loss/core": 0.16569259762763977,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.3934152126312256,
|
|
"rewards/margins": 1.474151611328125,
|
|
"rewards/rejected": 0.9192636609077454,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35810205908683973,
|
|
"grad_norm": 19.0,
|
|
"ht_mnpo/logit": 0.2867982089519501,
|
|
"ht_mnpo/residual": 0.27929818630218506,
|
|
"ht_mnpo/residual_abs": 0.2815331816673279,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -1.7822672128677368,
|
|
"logits/rejected": -2.033956289291382,
|
|
"logps/chosen": -0.2669016718864441,
|
|
"logps/rejected": -0.2721244990825653,
|
|
"loss": 0.7982349991798401,
|
|
"loss/core": 0.7982349991798401,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.7743923664093018,
|
|
"rewards/margins": 2.8679816722869873,
|
|
"rewards/rejected": 0.9064106941223145,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.3628767532079976,
|
|
"grad_norm": 12.4375,
|
|
"ht_mnpo/logit": 0.22812096774578094,
|
|
"ht_mnpo/residual": 0.22062095999717712,
|
|
"ht_mnpo/residual_abs": 0.22334913909435272,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -1.9232280254364014,
|
|
"logits/rejected": -2.178192615509033,
|
|
"logps/chosen": -0.29933086037635803,
|
|
"logps/rejected": -0.28768283128738403,
|
|
"loss": 0.6772836446762085,
|
|
"loss/core": 0.6772836446762085,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.9951553344726562,
|
|
"rewards/margins": 2.281209945678711,
|
|
"rewards/rejected": 0.713945746421814,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.3676514473291555,
|
|
"grad_norm": 3.25,
|
|
"ht_mnpo/logit": 0.21591122448444366,
|
|
"ht_mnpo/residual": 0.20841124653816223,
|
|
"ht_mnpo/residual_abs": 0.21050401031970978,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -1.847633719444275,
|
|
"logits/rejected": -2.065316677093506,
|
|
"logps/chosen": -0.28148138523101807,
|
|
"logps/rejected": -0.2986345589160919,
|
|
"loss": 0.47226905822753906,
|
|
"loss/core": 0.47226905822753906,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.5782718658447266,
|
|
"rewards/margins": 2.1591124534606934,
|
|
"rewards/rejected": 0.4191594123840332,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.37242614145031333,
|
|
"grad_norm": 404.0,
|
|
"ht_mnpo/logit": 0.4722256064414978,
|
|
"ht_mnpo/residual": 0.46472564339637756,
|
|
"ht_mnpo/residual_abs": 0.46695786714553833,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.9401696920394897,
|
|
"logits/rejected": -2.1430230140686035,
|
|
"logps/chosen": -0.3209291100502014,
|
|
"logps/rejected": -0.29362010955810547,
|
|
"loss": 2.3541719913482666,
|
|
"loss/core": 2.3541719913482666,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 5.9721999168396,
|
|
"rewards/margins": 4.722256183624268,
|
|
"rewards/rejected": 1.2499440908432007,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.3772008355714712,
|
|
"grad_norm": 28.75,
|
|
"ht_mnpo/logit": 0.2606656849384308,
|
|
"ht_mnpo/residual": 0.25316569209098816,
|
|
"ht_mnpo/residual_abs": 0.2560683488845825,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -1.8940608501434326,
|
|
"logits/rejected": -2.079465389251709,
|
|
"logps/chosen": -0.2855537533760071,
|
|
"logps/rejected": -0.2773713767528534,
|
|
"loss": 0.5645236372947693,
|
|
"loss/core": 0.5645236372947693,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.6368789672851562,
|
|
"rewards/margins": 2.606656789779663,
|
|
"rewards/rejected": 1.030221700668335,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38197552969262905,
|
|
"grad_norm": 187.0,
|
|
"ht_mnpo/logit": 0.26339191198349,
|
|
"ht_mnpo/residual": 0.25589194893836975,
|
|
"ht_mnpo/residual_abs": 0.2572072148323059,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.0253453254699707,
|
|
"logits/rejected": -2.415069818496704,
|
|
"logps/chosen": -0.2948891520500183,
|
|
"logps/rejected": -0.28987759351730347,
|
|
"loss": 0.494838148355484,
|
|
"loss/core": 0.494838148355484,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.4587039947509766,
|
|
"rewards/margins": 2.6339192390441895,
|
|
"rewards/rejected": 0.824785053730011,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.38675022381378693,
|
|
"grad_norm": 796.0,
|
|
"ht_mnpo/logit": 0.519372820854187,
|
|
"ht_mnpo/residual": 0.5118728876113892,
|
|
"ht_mnpo/residual_abs": 0.5148923993110657,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -1.5729597806930542,
|
|
"logits/rejected": -1.9059674739837646,
|
|
"logps/chosen": -0.324828565120697,
|
|
"logps/rejected": -0.3063935339450836,
|
|
"loss": 2.9073703289031982,
|
|
"loss/core": 2.9073703289031982,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 6.127835750579834,
|
|
"rewards/margins": 5.193729400634766,
|
|
"rewards/rejected": 0.9341069459915161,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.3915249179349448,
|
|
"grad_norm": 428.0,
|
|
"ht_mnpo/logit": 0.33313632011413574,
|
|
"ht_mnpo/residual": 0.3256363272666931,
|
|
"ht_mnpo/residual_abs": 0.3284788727760315,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -1.9473098516464233,
|
|
"logits/rejected": -2.265756130218506,
|
|
"logps/chosen": -0.28903454542160034,
|
|
"logps/rejected": -0.29171279072761536,
|
|
"loss": 1.68734610080719,
|
|
"loss/core": 1.68734610080719,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.016785621643066,
|
|
"rewards/margins": 3.3313636779785156,
|
|
"rewards/rejected": 0.6854226589202881,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.39629961205610265,
|
|
"grad_norm": 4.96875,
|
|
"ht_mnpo/logit": 0.14160475134849548,
|
|
"ht_mnpo/residual": 0.13410475850105286,
|
|
"ht_mnpo/residual_abs": 0.1367226243019104,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -2.0103304386138916,
|
|
"logits/rejected": -2.196492910385132,
|
|
"logps/chosen": -0.2817983031272888,
|
|
"logps/rejected": -0.30435535311698914,
|
|
"loss": 0.11378359794616699,
|
|
"loss/core": 0.11378359794616699,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.1162495613098145,
|
|
"rewards/margins": 1.41604745388031,
|
|
"rewards/rejected": 0.7002020478248596,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40107430617726053,
|
|
"grad_norm": 792.0,
|
|
"ht_mnpo/logit": 0.2766280174255371,
|
|
"ht_mnpo/residual": 0.2691280245780945,
|
|
"ht_mnpo/residual_abs": 0.27129045128822327,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -1.7459360361099243,
|
|
"logits/rejected": -1.9585685729980469,
|
|
"logps/chosen": -0.2827063500881195,
|
|
"logps/rejected": -0.2677227854728699,
|
|
"loss": 0.8373321294784546,
|
|
"loss/core": 0.8373321294784546,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.167675495147705,
|
|
"rewards/margins": 2.766279935836792,
|
|
"rewards/rejected": 1.401395320892334,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.40584900029841836,
|
|
"grad_norm": 676.0,
|
|
"ht_mnpo/logit": 0.22410964965820312,
|
|
"ht_mnpo/residual": 0.2166096419095993,
|
|
"ht_mnpo/residual_abs": 0.22375044226646423,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -1.9364029169082642,
|
|
"logits/rejected": -2.2243618965148926,
|
|
"logps/chosen": -0.29613256454467773,
|
|
"logps/rejected": -0.2813289761543274,
|
|
"loss": 0.44919532537460327,
|
|
"loss/core": 0.44919532537460327,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.634718418121338,
|
|
"rewards/margins": 2.2410967350006104,
|
|
"rewards/rejected": 1.3936221599578857,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41062369441957625,
|
|
"grad_norm": 2.953125,
|
|
"ht_mnpo/logit": 0.11446923017501831,
|
|
"ht_mnpo/residual": 0.10696922242641449,
|
|
"ht_mnpo/residual_abs": 0.10941126197576523,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.0317301750183105,
|
|
"logits/rejected": -2.2824201583862305,
|
|
"logps/chosen": -0.28097397089004517,
|
|
"logps/rejected": -0.2934049665927887,
|
|
"loss": 0.04260837286710739,
|
|
"loss/core": 0.04260837286710739,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.770204782485962,
|
|
"rewards/margins": 1.1446921825408936,
|
|
"rewards/rejected": 0.6255124807357788,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41539838854073413,
|
|
"grad_norm": 6.75,
|
|
"ht_mnpo/logit": 0.17668896913528442,
|
|
"ht_mnpo/residual": 0.1691889464855194,
|
|
"ht_mnpo/residual_abs": 0.17058250308036804,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.0280332565307617,
|
|
"logits/rejected": -2.2525267601013184,
|
|
"logps/chosen": -0.2677651345729828,
|
|
"logps/rejected": -0.2582930326461792,
|
|
"loss": 0.2649860978126526,
|
|
"loss/core": 0.2649860978126526,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.587427854537964,
|
|
"rewards/margins": 1.7668898105621338,
|
|
"rewards/rejected": 0.8205384016036987,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.42017308266189196,
|
|
"grad_norm": 18.25,
|
|
"ht_mnpo/logit": 0.146823450922966,
|
|
"ht_mnpo/residual": 0.13932344317436218,
|
|
"ht_mnpo/residual_abs": 0.1414867490530014,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -1.963629961013794,
|
|
"logits/rejected": -2.183755397796631,
|
|
"logps/chosen": -0.2657316327095032,
|
|
"logps/rejected": -0.2711542248725891,
|
|
"loss": 0.07426687330007553,
|
|
"loss/core": 0.07426687330007553,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.3465564250946045,
|
|
"rewards/margins": 1.4682343006134033,
|
|
"rewards/rejected": 0.878322422504425,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42494777678304985,
|
|
"grad_norm": 3.625,
|
|
"ht_mnpo/logit": 0.2173123061656952,
|
|
"ht_mnpo/residual": 0.20981231331825256,
|
|
"ht_mnpo/residual_abs": 0.21300652623176575,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.9978408813476562,
|
|
"logits/rejected": -2.254274606704712,
|
|
"logps/chosen": -0.3106047511100769,
|
|
"logps/rejected": -0.28541427850723267,
|
|
"loss": 1.295969843864441,
|
|
"loss/core": 1.295969843864441,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9272522926330566,
|
|
"rewards/margins": 2.1731228828430176,
|
|
"rewards/rejected": 0.7541292905807495,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4297224709042077,
|
|
"grad_norm": 1128.0,
|
|
"ht_mnpo/logit": 0.2916508913040161,
|
|
"ht_mnpo/residual": 0.2841508984565735,
|
|
"ht_mnpo/residual_abs": 0.28610268235206604,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.9271923303604126,
|
|
"logits/rejected": -2.2027246952056885,
|
|
"logps/chosen": -0.3097872734069824,
|
|
"logps/rejected": -0.28542861342430115,
|
|
"loss": 0.8190312385559082,
|
|
"loss/core": 0.8190312385559082,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.744999408721924,
|
|
"rewards/margins": 2.9165091514587402,
|
|
"rewards/rejected": 0.8284900784492493,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.43449716502536556,
|
|
"grad_norm": 6.40625,
|
|
"ht_mnpo/logit": 0.21481645107269287,
|
|
"ht_mnpo/residual": 0.20731644332408905,
|
|
"ht_mnpo/residual_abs": 0.2103075236082077,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.8237874507904053,
|
|
"logits/rejected": -2.129227638244629,
|
|
"logps/chosen": -0.30449140071868896,
|
|
"logps/rejected": -0.27840521931648254,
|
|
"loss": 0.42009902000427246,
|
|
"loss/core": 0.42009902000427246,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.241947889328003,
|
|
"rewards/margins": 2.1481645107269287,
|
|
"rewards/rejected": 1.0937836170196533,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.43927185914652345,
|
|
"grad_norm": 167.0,
|
|
"ht_mnpo/logit": 0.29557928442955017,
|
|
"ht_mnpo/residual": 0.28807929158210754,
|
|
"ht_mnpo/residual_abs": 0.2902158796787262,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -1.9467674493789673,
|
|
"logits/rejected": -2.1150288581848145,
|
|
"logps/chosen": -0.25731417536735535,
|
|
"logps/rejected": -0.2749153673648834,
|
|
"loss": 1.0729789733886719,
|
|
"loss/core": 1.0729789733886719,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.7630081176757812,
|
|
"rewards/margins": 2.9557926654815674,
|
|
"rewards/rejected": 0.8072150945663452,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.4440465532676813,
|
|
"grad_norm": 5.28125,
|
|
"ht_mnpo/logit": 0.1232837587594986,
|
|
"ht_mnpo/residual": 0.11578376591205597,
|
|
"ht_mnpo/residual_abs": 0.11795320361852646,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.9201545715332031,
|
|
"logits/rejected": -2.2683331966400146,
|
|
"logps/chosen": -0.3000238239765167,
|
|
"logps/rejected": -0.3087528347969055,
|
|
"loss": 0.0906161218881607,
|
|
"loss/core": 0.0906161218881607,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7928030490875244,
|
|
"rewards/margins": 1.2328376770019531,
|
|
"rewards/rejected": 0.5599652528762817,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.44882124738883916,
|
|
"grad_norm": 55.5,
|
|
"ht_mnpo/logit": 0.3084077537059784,
|
|
"ht_mnpo/residual": 0.3009077310562134,
|
|
"ht_mnpo/residual_abs": 0.3186038136482239,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -1.9410159587860107,
|
|
"logits/rejected": -2.1513659954071045,
|
|
"logps/chosen": -0.3083992600440979,
|
|
"logps/rejected": -0.3033507764339447,
|
|
"loss": 1.498625636100769,
|
|
"loss/core": 1.498625636100769,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.285151481628418,
|
|
"rewards/margins": 3.084077835083008,
|
|
"rewards/rejected": 1.2010738849639893,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453595941509997,
|
|
"grad_norm": 352.0,
|
|
"ht_mnpo/logit": 0.36416906118392944,
|
|
"ht_mnpo/residual": 0.35666903853416443,
|
|
"ht_mnpo/residual_abs": 0.3577561676502228,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.192099094390869,
|
|
"logits/rejected": -2.485234498977661,
|
|
"logps/chosen": -0.29125604033470154,
|
|
"logps/rejected": -0.3030247092247009,
|
|
"loss": 1.63297438621521,
|
|
"loss/core": 1.63297438621521,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 4.850213050842285,
|
|
"rewards/margins": 3.641690492630005,
|
|
"rewards/rejected": 1.2085225582122803,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.4583706356311549,
|
|
"grad_norm": 15.0,
|
|
"ht_mnpo/logit": 0.3097284436225891,
|
|
"ht_mnpo/residual": 0.30222851037979126,
|
|
"ht_mnpo/residual_abs": 0.3041606545448303,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -1.9110229015350342,
|
|
"logits/rejected": -2.150383472442627,
|
|
"logps/chosen": -0.29265767335891724,
|
|
"logps/rejected": -0.28647303581237793,
|
|
"loss": 0.8241177797317505,
|
|
"loss/core": 0.8241177797317505,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.338518142700195,
|
|
"rewards/margins": 3.0972847938537598,
|
|
"rewards/rejected": 1.241233229637146,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.46314532975231276,
|
|
"grad_norm": 238.0,
|
|
"ht_mnpo/logit": 0.3325381875038147,
|
|
"ht_mnpo/residual": 0.32503825426101685,
|
|
"ht_mnpo/residual_abs": 0.3343810439109802,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -1.8738826513290405,
|
|
"logits/rejected": -2.16226863861084,
|
|
"logps/chosen": -0.3215123116970062,
|
|
"logps/rejected": -0.3182523846626282,
|
|
"loss": 1.0384169816970825,
|
|
"loss/core": 1.0384169816970825,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.18947696685791,
|
|
"rewards/margins": 3.3253822326660156,
|
|
"rewards/rejected": 0.8640945553779602,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4679200238734706,
|
|
"grad_norm": 12.5,
|
|
"ht_mnpo/logit": 0.14976148307323456,
|
|
"ht_mnpo/residual": 0.14226147532463074,
|
|
"ht_mnpo/residual_abs": 0.14404740929603577,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.215057849884033,
|
|
"logits/rejected": -2.4560017585754395,
|
|
"logps/chosen": -0.2996280789375305,
|
|
"logps/rejected": -0.30207791924476624,
|
|
"loss": 0.15293726325035095,
|
|
"loss/core": 0.15293726325035095,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.62994384765625,
|
|
"rewards/margins": 1.497614860534668,
|
|
"rewards/rejected": 1.1323288679122925,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.4726947179946285,
|
|
"grad_norm": 1.109375,
|
|
"ht_mnpo/logit": 0.1131284236907959,
|
|
"ht_mnpo/residual": 0.10562840849161148,
|
|
"ht_mnpo/residual_abs": 0.10809159278869629,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.2300727367401123,
|
|
"logits/rejected": -2.570033550262451,
|
|
"logps/chosen": -0.2869029641151428,
|
|
"logps/rejected": -0.2894020676612854,
|
|
"loss": 0.0645766630768776,
|
|
"loss/core": 0.0645766630768776,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0836386680603027,
|
|
"rewards/margins": 1.1312841176986694,
|
|
"rewards/rejected": 0.9523545503616333,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.4774694121157863,
|
|
"grad_norm": 384.0,
|
|
"ht_mnpo/logit": 0.1857949197292328,
|
|
"ht_mnpo/residual": 0.17829494178295135,
|
|
"ht_mnpo/residual_abs": 0.1815871000289917,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.7350391149520874,
|
|
"logits/rejected": -2.03250789642334,
|
|
"logps/chosen": -0.30827873945236206,
|
|
"logps/rejected": -0.30130141973495483,
|
|
"loss": 0.2996276617050171,
|
|
"loss/core": 0.2996276617050171,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.6126999855041504,
|
|
"rewards/margins": 1.8579492568969727,
|
|
"rewards/rejected": 0.7547506093978882,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4822441062369442,
|
|
"grad_norm": 11.75,
|
|
"ht_mnpo/logit": 0.196578711271286,
|
|
"ht_mnpo/residual": 0.18907873332500458,
|
|
"ht_mnpo/residual_abs": 0.19180910289287567,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -1.9780470132827759,
|
|
"logits/rejected": -2.1910805702209473,
|
|
"logps/chosen": -0.31306812167167664,
|
|
"logps/rejected": -0.31634530425071716,
|
|
"loss": 0.27675363421440125,
|
|
"loss/core": 0.27675363421440125,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.940455913543701,
|
|
"rewards/margins": 1.9657869338989258,
|
|
"rewards/rejected": 0.9746688008308411,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4870188003581021,
|
|
"grad_norm": 11.6875,
|
|
"ht_mnpo/logit": 0.29334843158721924,
|
|
"ht_mnpo/residual": 0.2858484089374542,
|
|
"ht_mnpo/residual_abs": 0.28805553913116455,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -1.839543104171753,
|
|
"logits/rejected": -2.139441728591919,
|
|
"logps/chosen": -0.2953009009361267,
|
|
"logps/rejected": -0.27880555391311646,
|
|
"loss": 0.7938116788864136,
|
|
"loss/core": 0.7938116788864136,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.985734462738037,
|
|
"rewards/margins": 2.9334845542907715,
|
|
"rewards/rejected": 1.0522500276565552,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.4917934944792599,
|
|
"grad_norm": 2.59375,
|
|
"ht_mnpo/logit": 0.11522988229990005,
|
|
"ht_mnpo/residual": 0.10772988945245743,
|
|
"ht_mnpo/residual_abs": 0.11010618507862091,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -1.898514747619629,
|
|
"logits/rejected": -2.2502241134643555,
|
|
"logps/chosen": -0.2737577557563782,
|
|
"logps/rejected": -0.2691265940666199,
|
|
"loss": 0.06796002388000488,
|
|
"loss/core": 0.06796002388000488,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.908595323562622,
|
|
"rewards/margins": 1.1522986888885498,
|
|
"rewards/rejected": 0.7562967538833618,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4965681886004178,
|
|
"grad_norm": 2.53125,
|
|
"ht_mnpo/logit": 0.13243892788887024,
|
|
"ht_mnpo/residual": 0.12493894249200821,
|
|
"ht_mnpo/residual_abs": 0.13076035678386688,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -1.9941246509552002,
|
|
"logits/rejected": -2.2544379234313965,
|
|
"logps/chosen": -0.2522723078727722,
|
|
"logps/rejected": -0.2937101721763611,
|
|
"loss": 0.08826544135808945,
|
|
"loss/core": 0.08826544135808945,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3944621086120605,
|
|
"rewards/margins": 1.3243893384933472,
|
|
"rewards/rejected": 1.0700726509094238,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5013428827215757,
|
|
"grad_norm": 5.6875,
|
|
"ht_mnpo/logit": 0.12501785159111023,
|
|
"ht_mnpo/residual": 0.117517851293087,
|
|
"ht_mnpo/residual_abs": 0.11929130554199219,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.2246017456054688,
|
|
"logits/rejected": -2.410853624343872,
|
|
"logps/chosen": -0.28422626852989197,
|
|
"logps/rejected": -0.28308025002479553,
|
|
"loss": 0.10617043823003769,
|
|
"loss/core": 0.10617043823003769,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.355593681335449,
|
|
"rewards/margins": 1.2501784563064575,
|
|
"rewards/rejected": 1.1054149866104126,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5061175768427335,
|
|
"grad_norm": 34.25,
|
|
"ht_mnpo/logit": 0.17451244592666626,
|
|
"ht_mnpo/residual": 0.16701245307922363,
|
|
"ht_mnpo/residual_abs": 0.16843153536319733,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.0963895320892334,
|
|
"logits/rejected": -2.3358659744262695,
|
|
"logps/chosen": -0.28125277161598206,
|
|
"logps/rejected": -0.2825583219528198,
|
|
"loss": 0.38109883666038513,
|
|
"loss/core": 0.38109883666038513,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.497371196746826,
|
|
"rewards/margins": 1.7451244592666626,
|
|
"rewards/rejected": 0.752246618270874,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5108922709638913,
|
|
"grad_norm": 68.5,
|
|
"ht_mnpo/logit": 0.2884078621864319,
|
|
"ht_mnpo/residual": 0.28090786933898926,
|
|
"ht_mnpo/residual_abs": 0.2828427851200104,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -1.8652563095092773,
|
|
"logits/rejected": -2.1189205646514893,
|
|
"logps/chosen": -0.3185581564903259,
|
|
"logps/rejected": -0.3078560531139374,
|
|
"loss": 0.5867989659309387,
|
|
"loss/core": 0.5867989659309387,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.6412525177001953,
|
|
"rewards/margins": 2.8840785026550293,
|
|
"rewards/rejected": 0.7571739554405212,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5156669650850493,
|
|
"grad_norm": 6.09375,
|
|
"ht_mnpo/logit": 0.10569143295288086,
|
|
"ht_mnpo/residual": 0.09819144755601883,
|
|
"ht_mnpo/residual_abs": 0.10155366361141205,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.0045483112335205,
|
|
"logits/rejected": -2.270784854888916,
|
|
"logps/chosen": -0.31180429458618164,
|
|
"logps/rejected": -0.3156093955039978,
|
|
"loss": 0.04411657899618149,
|
|
"loss/core": 0.04411657899618149,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5413364171981812,
|
|
"rewards/margins": 1.0569144487380981,
|
|
"rewards/rejected": 0.48442205786705017,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.5204416592062071,
|
|
"grad_norm": 580.0,
|
|
"ht_mnpo/logit": 0.19196265935897827,
|
|
"ht_mnpo/residual": 0.18446265161037445,
|
|
"ht_mnpo/residual_abs": 0.186975359916687,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.0267083644866943,
|
|
"logits/rejected": -2.356262683868408,
|
|
"logps/chosen": -0.29003921151161194,
|
|
"logps/rejected": -0.3435795307159424,
|
|
"loss": 0.3793261647224426,
|
|
"loss/core": 0.3793261647224426,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.015592575073242,
|
|
"rewards/margins": 1.9196265935897827,
|
|
"rewards/rejected": 1.095966100692749,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5252163533273649,
|
|
"grad_norm": 7.8125,
|
|
"ht_mnpo/logit": 0.23898443579673767,
|
|
"ht_mnpo/residual": 0.23148444294929504,
|
|
"ht_mnpo/residual_abs": 0.2341163158416748,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.8253529071807861,
|
|
"logits/rejected": -2.06758713722229,
|
|
"logps/chosen": -0.27169501781463623,
|
|
"logps/rejected": -0.2784484326839447,
|
|
"loss": 0.5674008131027222,
|
|
"loss/core": 0.5674008131027222,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5318641662597656,
|
|
"rewards/margins": 2.3898441791534424,
|
|
"rewards/rejected": 1.142019510269165,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5299910474485229,
|
|
"grad_norm": 2.96875,
|
|
"ht_mnpo/logit": 0.19051536917686462,
|
|
"ht_mnpo/residual": 0.183015376329422,
|
|
"ht_mnpo/residual_abs": 0.18519142270088196,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.0828499794006348,
|
|
"logits/rejected": -2.2646288871765137,
|
|
"logps/chosen": -0.31175822019577026,
|
|
"logps/rejected": -0.28370457887649536,
|
|
"loss": 0.45988917350769043,
|
|
"loss/core": 0.45988917350769043,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.769913911819458,
|
|
"rewards/margins": 1.9051536321640015,
|
|
"rewards/rejected": 0.8647602200508118,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5347657415696807,
|
|
"grad_norm": 2.28125,
|
|
"ht_mnpo/logit": 0.17307063937187195,
|
|
"ht_mnpo/residual": 0.16557064652442932,
|
|
"ht_mnpo/residual_abs": 0.16744212806224823,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -1.9716126918792725,
|
|
"logits/rejected": -2.2994656562805176,
|
|
"logps/chosen": -0.301216185092926,
|
|
"logps/rejected": -0.27583974599838257,
|
|
"loss": 0.45512303709983826,
|
|
"loss/core": 0.45512303709983826,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 2.420431137084961,
|
|
"rewards/margins": 1.7307065725326538,
|
|
"rewards/rejected": 0.6897248029708862,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5395404356908385,
|
|
"grad_norm": 4.4375,
|
|
"ht_mnpo/logit": 0.34825602173805237,
|
|
"ht_mnpo/residual": 0.34075599908828735,
|
|
"ht_mnpo/residual_abs": 0.34296515583992004,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.1434173583984375,
|
|
"logits/rejected": -2.416426420211792,
|
|
"logps/chosen": -0.3156982362270355,
|
|
"logps/rejected": -0.2996530830860138,
|
|
"loss": 2.206251382827759,
|
|
"loss/core": 2.206251382827759,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.403435707092285,
|
|
"rewards/margins": 3.4825599193573,
|
|
"rewards/rejected": 0.9208753705024719,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5443151298119964,
|
|
"grad_norm": 60.0,
|
|
"ht_mnpo/logit": 0.24878111481666565,
|
|
"ht_mnpo/residual": 0.24128107726573944,
|
|
"ht_mnpo/residual_abs": 0.246558278799057,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -1.9773708581924438,
|
|
"logits/rejected": -2.3607583045959473,
|
|
"logps/chosen": -0.28530240058898926,
|
|
"logps/rejected": -0.27613160014152527,
|
|
"loss": 0.4744301736354828,
|
|
"loss/core": 0.4744301736354828,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.3544468879699707,
|
|
"rewards/margins": 2.487811326980591,
|
|
"rewards/rejected": 0.8666356801986694,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5490898239331543,
|
|
"grad_norm": 2.078125,
|
|
"ht_mnpo/logit": 0.31033894419670105,
|
|
"ht_mnpo/residual": 0.3028389513492584,
|
|
"ht_mnpo/residual_abs": 0.30536240339279175,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.0291919708251953,
|
|
"logits/rejected": -2.281064510345459,
|
|
"logps/chosen": -0.32846799492836,
|
|
"logps/rejected": -0.3118685483932495,
|
|
"loss": 1.0607415437698364,
|
|
"loss/core": 1.0607415437698364,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.091516971588135,
|
|
"rewards/margins": 3.1033897399902344,
|
|
"rewards/rejected": 0.9881270527839661,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5538645180543121,
|
|
"grad_norm": 3.6875,
|
|
"ht_mnpo/logit": 0.4819414019584656,
|
|
"ht_mnpo/residual": 0.47444137930870056,
|
|
"ht_mnpo/residual_abs": 0.4764309823513031,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -1.7918148040771484,
|
|
"logits/rejected": -1.9814106225967407,
|
|
"logps/chosen": -0.30701881647109985,
|
|
"logps/rejected": -0.25892844796180725,
|
|
"loss": 2.3531346321105957,
|
|
"loss/core": 2.3531346321105957,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 6.004878520965576,
|
|
"rewards/margins": 4.819413661956787,
|
|
"rewards/rejected": 1.18546462059021,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.55863921217547,
|
|
"grad_norm": 50.75,
|
|
"ht_mnpo/logit": 0.156865194439888,
|
|
"ht_mnpo/residual": 0.14936518669128418,
|
|
"ht_mnpo/residual_abs": 0.15104547142982483,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -2.02787709236145,
|
|
"logits/rejected": -2.223013401031494,
|
|
"logps/chosen": -0.29150041937828064,
|
|
"logps/rejected": -0.2901824414730072,
|
|
"loss": 0.09419430792331696,
|
|
"loss/core": 0.09419430792331696,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.5935516357421875,
|
|
"rewards/margins": 1.5686519145965576,
|
|
"rewards/rejected": 1.0248996019363403,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5634139062966279,
|
|
"grad_norm": 888.0,
|
|
"ht_mnpo/logit": 0.2369958609342575,
|
|
"ht_mnpo/residual": 0.2294958084821701,
|
|
"ht_mnpo/residual_abs": 0.2326071560382843,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -1.8807750940322876,
|
|
"logits/rejected": -2.03248929977417,
|
|
"logps/chosen": -0.29985108971595764,
|
|
"logps/rejected": -0.3031771183013916,
|
|
"loss": 0.7982430458068848,
|
|
"loss/core": 0.7982430458068848,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 3.11862850189209,
|
|
"rewards/margins": 2.3699584007263184,
|
|
"rewards/rejected": 0.7486701011657715,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5681886004177857,
|
|
"grad_norm": 2.515625,
|
|
"ht_mnpo/logit": 0.4987797141075134,
|
|
"ht_mnpo/residual": 0.4912796914577484,
|
|
"ht_mnpo/residual_abs": 0.4936184287071228,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -1.8091611862182617,
|
|
"logits/rejected": -2.182551622390747,
|
|
"logps/chosen": -0.3149603307247162,
|
|
"logps/rejected": -0.2931450605392456,
|
|
"loss": 2.4295339584350586,
|
|
"loss/core": 2.4295339584350586,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 5.907982349395752,
|
|
"rewards/margins": 4.987797260284424,
|
|
"rewards/rejected": 0.9201849102973938,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5729632945389436,
|
|
"grad_norm": 2.09375,
|
|
"ht_mnpo/logit": 0.24387526512145996,
|
|
"ht_mnpo/residual": 0.23637528717517853,
|
|
"ht_mnpo/residual_abs": 0.23978765308856964,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -1.8736908435821533,
|
|
"logits/rejected": -2.1749606132507324,
|
|
"logps/chosen": -0.29759305715560913,
|
|
"logps/rejected": -0.2989031970500946,
|
|
"loss": 0.559294581413269,
|
|
"loss/core": 0.559294581413269,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.058932304382324,
|
|
"rewards/margins": 2.4387524127960205,
|
|
"rewards/rejected": 1.6201797723770142,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5777379886601015,
|
|
"grad_norm": 16.375,
|
|
"ht_mnpo/logit": 0.27251681685447693,
|
|
"ht_mnpo/residual": 0.2650168240070343,
|
|
"ht_mnpo/residual_abs": 0.2667366862297058,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -1.8784462213516235,
|
|
"logits/rejected": -2.1159400939941406,
|
|
"logps/chosen": -0.29141777753829956,
|
|
"logps/rejected": -0.3019111752510071,
|
|
"loss": 0.662787914276123,
|
|
"loss/core": 0.662787914276123,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.4891610145568848,
|
|
"rewards/margins": 2.725167989730835,
|
|
"rewards/rejected": 0.7639929056167603,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5825126827812593,
|
|
"grad_norm": 3.5625,
|
|
"ht_mnpo/logit": 0.23614993691444397,
|
|
"ht_mnpo/residual": 0.22864992916584015,
|
|
"ht_mnpo/residual_abs": 0.23106643557548523,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -1.9317668676376343,
|
|
"logits/rejected": -2.19699764251709,
|
|
"logps/chosen": -0.301189124584198,
|
|
"logps/rejected": -0.2828231751918793,
|
|
"loss": 0.7685016393661499,
|
|
"loss/core": 0.7685016393661499,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8618009090423584,
|
|
"rewards/margins": 2.361499309539795,
|
|
"rewards/rejected": 0.5003016591072083,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.5872873769024172,
|
|
"grad_norm": 11.1875,
|
|
"ht_mnpo/logit": 0.3057871460914612,
|
|
"ht_mnpo/residual": 0.2982870936393738,
|
|
"ht_mnpo/residual_abs": 0.30613571405410767,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.7794278860092163,
|
|
"logits/rejected": -2.1755802631378174,
|
|
"logps/chosen": -0.2987704873085022,
|
|
"logps/rejected": -0.2674489915370941,
|
|
"loss": 1.0454192161560059,
|
|
"loss/core": 1.0454192161560059,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.152740001678467,
|
|
"rewards/margins": 3.0578713417053223,
|
|
"rewards/rejected": 1.094868779182434,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.592062071023575,
|
|
"grad_norm": 7.90625,
|
|
"ht_mnpo/logit": 0.16309209167957306,
|
|
"ht_mnpo/residual": 0.15559211373329163,
|
|
"ht_mnpo/residual_abs": 0.1573321521282196,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.0946414470672607,
|
|
"logits/rejected": -2.3167364597320557,
|
|
"logps/chosen": -0.2965165972709656,
|
|
"logps/rejected": -0.2895964980125427,
|
|
"loss": 0.220075324177742,
|
|
"loss/core": 0.220075324177742,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.6535112857818604,
|
|
"rewards/margins": 1.6309210062026978,
|
|
"rewards/rejected": 1.0225903987884521,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5968367651447329,
|
|
"grad_norm": 168.0,
|
|
"ht_mnpo/logit": 0.26800763607025146,
|
|
"ht_mnpo/residual": 0.26050764322280884,
|
|
"ht_mnpo/residual_abs": 0.26258984208106995,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.8491462469100952,
|
|
"logits/rejected": -2.066373586654663,
|
|
"logps/chosen": -0.2881318926811218,
|
|
"logps/rejected": -0.30786803364753723,
|
|
"loss": 0.5212100744247437,
|
|
"loss/core": 0.5212100744247437,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.430001735687256,
|
|
"rewards/margins": 2.6800761222839355,
|
|
"rewards/rejected": 0.7499252557754517,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6016114592658908,
|
|
"grad_norm": 190.0,
|
|
"ht_mnpo/logit": 0.3151053786277771,
|
|
"ht_mnpo/residual": 0.3076053857803345,
|
|
"ht_mnpo/residual_abs": 0.3096749186515808,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.9944003820419312,
|
|
"logits/rejected": -2.25925350189209,
|
|
"logps/chosen": -0.33351221680641174,
|
|
"logps/rejected": -0.30969151854515076,
|
|
"loss": 1.3126461505889893,
|
|
"loss/core": 1.3126461505889893,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 4.146966457366943,
|
|
"rewards/margins": 3.1510536670684814,
|
|
"rewards/rejected": 0.9959124326705933,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6063861533870486,
|
|
"grad_norm": 2.15625,
|
|
"ht_mnpo/logit": 0.16642612218856812,
|
|
"ht_mnpo/residual": 0.15892614424228668,
|
|
"ht_mnpo/residual_abs": 0.1607307344675064,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -2.1082916259765625,
|
|
"logits/rejected": -2.336918354034424,
|
|
"logps/chosen": -0.28919678926467896,
|
|
"logps/rejected": -0.2757853865623474,
|
|
"loss": 0.32006698846817017,
|
|
"loss/core": 0.32006698846817017,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.2848565578460693,
|
|
"rewards/margins": 1.6642612218856812,
|
|
"rewards/rejected": 0.6205950975418091,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6111608475082065,
|
|
"grad_norm": 188.0,
|
|
"ht_mnpo/logit": 0.13086314499378204,
|
|
"ht_mnpo/residual": 0.12336313724517822,
|
|
"ht_mnpo/residual_abs": 0.1262132078409195,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.047877550125122,
|
|
"logits/rejected": -2.198028564453125,
|
|
"logps/chosen": -0.30865031480789185,
|
|
"logps/rejected": -0.29963168501853943,
|
|
"loss": 0.12744709849357605,
|
|
"loss/core": 0.12744709849357605,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.1879143714904785,
|
|
"rewards/margins": 1.308631420135498,
|
|
"rewards/rejected": 0.87928307056427,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6159355416293644,
|
|
"grad_norm": 12.75,
|
|
"ht_mnpo/logit": 0.35795003175735474,
|
|
"ht_mnpo/residual": 0.3504500985145569,
|
|
"ht_mnpo/residual_abs": 0.3530126214027405,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -1.7279043197631836,
|
|
"logits/rejected": -2.076632022857666,
|
|
"logps/chosen": -0.309658944606781,
|
|
"logps/rejected": -0.2867974638938904,
|
|
"loss": 1.6379516124725342,
|
|
"loss/core": 1.6379516124725342,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 5.011632919311523,
|
|
"rewards/margins": 3.579500675201416,
|
|
"rewards/rejected": 1.4321321249008179,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6207102357505222,
|
|
"grad_norm": 78.5,
|
|
"ht_mnpo/logit": 0.2575874626636505,
|
|
"ht_mnpo/residual": 0.2500874698162079,
|
|
"ht_mnpo/residual_abs": 0.25217220187187195,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -1.9400348663330078,
|
|
"logits/rejected": -2.297685146331787,
|
|
"logps/chosen": -0.2735014259815216,
|
|
"logps/rejected": -0.2798306345939636,
|
|
"loss": 0.5699511170387268,
|
|
"loss/core": 0.5699511170387268,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.7315871715545654,
|
|
"rewards/margins": 2.5758748054504395,
|
|
"rewards/rejected": 1.1557127237319946,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6254849298716801,
|
|
"grad_norm": 4.4375,
|
|
"ht_mnpo/logit": 0.11060944944620132,
|
|
"ht_mnpo/residual": 0.1031094565987587,
|
|
"ht_mnpo/residual_abs": 0.10582210123538971,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.1760499477386475,
|
|
"logits/rejected": -2.4464058876037598,
|
|
"logps/chosen": -0.263247013092041,
|
|
"logps/rejected": -0.2721119225025177,
|
|
"loss": 0.043946050107479095,
|
|
"loss/core": 0.043946050107479095,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.126974105834961,
|
|
"rewards/margins": 1.1060945987701416,
|
|
"rewards/rejected": 1.0208795070648193,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.630259623992838,
|
|
"grad_norm": 286.0,
|
|
"ht_mnpo/logit": 0.3428904116153717,
|
|
"ht_mnpo/residual": 0.33539044857025146,
|
|
"ht_mnpo/residual_abs": 0.33829253911972046,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -1.8339145183563232,
|
|
"logits/rejected": -2.0472755432128906,
|
|
"logps/chosen": -0.33507493138313293,
|
|
"logps/rejected": -0.31190431118011475,
|
|
"loss": 1.6195424795150757,
|
|
"loss/core": 1.6195424795150757,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.361727237701416,
|
|
"rewards/margins": 3.4289040565490723,
|
|
"rewards/rejected": 0.9328228831291199,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6350343181139958,
|
|
"grad_norm": 3.875,
|
|
"ht_mnpo/logit": 0.3288431465625763,
|
|
"ht_mnpo/residual": 0.3213431239128113,
|
|
"ht_mnpo/residual_abs": 0.323604017496109,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -1.9675363302230835,
|
|
"logits/rejected": -2.2226428985595703,
|
|
"logps/chosen": -0.33497220277786255,
|
|
"logps/rejected": -0.2824486792087555,
|
|
"loss": 1.085584044456482,
|
|
"loss/core": 1.085584044456482,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.039675712585449,
|
|
"rewards/margins": 3.288431167602539,
|
|
"rewards/rejected": 0.7512443661689758,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.6398090122351537,
|
|
"grad_norm": 49.5,
|
|
"ht_mnpo/logit": 0.26619061827659607,
|
|
"ht_mnpo/residual": 0.25869062542915344,
|
|
"ht_mnpo/residual_abs": 0.2621300220489502,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.9468028545379639,
|
|
"logits/rejected": -2.044259548187256,
|
|
"logps/chosen": -0.32372915744781494,
|
|
"logps/rejected": -0.32650211453437805,
|
|
"loss": 1.0328400135040283,
|
|
"loss/core": 1.0328400135040283,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.102653503417969,
|
|
"rewards/margins": 2.6619060039520264,
|
|
"rewards/rejected": 1.440747618675232,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6445837063563116,
|
|
"grad_norm": 155.0,
|
|
"ht_mnpo/logit": 0.1919471025466919,
|
|
"ht_mnpo/residual": 0.18444707989692688,
|
|
"ht_mnpo/residual_abs": 0.18674826622009277,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.0553970336914062,
|
|
"logits/rejected": -2.277501344680786,
|
|
"logps/chosen": -0.30528566241264343,
|
|
"logps/rejected": -0.30857595801353455,
|
|
"loss": 0.44716134667396545,
|
|
"loss/core": 0.44716134667396545,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.826671838760376,
|
|
"rewards/margins": 1.9194707870483398,
|
|
"rewards/rejected": 0.9072009325027466,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6493584004774694,
|
|
"grad_norm": 1408.0,
|
|
"ht_mnpo/logit": 0.5156992077827454,
|
|
"ht_mnpo/residual": 0.5081992149353027,
|
|
"ht_mnpo/residual_abs": 0.5100849270820618,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -1.9224817752838135,
|
|
"logits/rejected": -2.295820713043213,
|
|
"logps/chosen": -0.3082597255706787,
|
|
"logps/rejected": -0.28871044516563416,
|
|
"loss": 2.1461853981018066,
|
|
"loss/core": 2.1461853981018066,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 6.397282600402832,
|
|
"rewards/margins": 5.1569929122924805,
|
|
"rewards/rejected": 1.2402900457382202,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.6541330945986272,
|
|
"grad_norm": 11.875,
|
|
"ht_mnpo/logit": 0.2202470600605011,
|
|
"ht_mnpo/residual": 0.21274705231189728,
|
|
"ht_mnpo/residual_abs": 0.21405646204948425,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -1.833820104598999,
|
|
"logits/rejected": -2.1766836643218994,
|
|
"logps/chosen": -0.2879013121128082,
|
|
"logps/rejected": -0.2877139449119568,
|
|
"loss": 0.39130714535713196,
|
|
"loss/core": 0.39130714535713196,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.120842456817627,
|
|
"rewards/margins": 2.202470541000366,
|
|
"rewards/rejected": 0.9183722734451294,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6589077887197852,
|
|
"grad_norm": 2.484375,
|
|
"ht_mnpo/logit": 0.33171364665031433,
|
|
"ht_mnpo/residual": 0.3242136240005493,
|
|
"ht_mnpo/residual_abs": 0.32709458470344543,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.9676786661148071,
|
|
"logits/rejected": -2.1830246448516846,
|
|
"logps/chosen": -0.31910890340805054,
|
|
"logps/rejected": -0.2641559839248657,
|
|
"loss": 1.0723878145217896,
|
|
"loss/core": 1.0723878145217896,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.9963231086730957,
|
|
"rewards/margins": 3.317136287689209,
|
|
"rewards/rejected": 0.6791867613792419,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.663682482840943,
|
|
"grad_norm": 54.75,
|
|
"ht_mnpo/logit": 0.35695138573646545,
|
|
"ht_mnpo/residual": 0.3494513928890228,
|
|
"ht_mnpo/residual_abs": 0.35167449712753296,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -1.966403603553772,
|
|
"logits/rejected": -2.119166851043701,
|
|
"logps/chosen": -0.3139706552028656,
|
|
"logps/rejected": -0.3095807731151581,
|
|
"loss": 1.3685941696166992,
|
|
"loss/core": 1.3685941696166992,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.306251525878906,
|
|
"rewards/margins": 3.569514513015747,
|
|
"rewards/rejected": 0.7367372512817383,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6684571769621008,
|
|
"grad_norm": 828.0,
|
|
"ht_mnpo/logit": 0.3428606390953064,
|
|
"ht_mnpo/residual": 0.335360586643219,
|
|
"ht_mnpo/residual_abs": 0.337501585483551,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -1.8964332342147827,
|
|
"logits/rejected": -2.292275905609131,
|
|
"logps/chosen": -0.30034637451171875,
|
|
"logps/rejected": -0.2862694263458252,
|
|
"loss": 1.527578592300415,
|
|
"loss/core": 1.527578592300415,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 4.155306816101074,
|
|
"rewards/margins": 3.428605556488037,
|
|
"rewards/rejected": 0.7267010807991028,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6732318710832588,
|
|
"grad_norm": 14.6875,
|
|
"ht_mnpo/logit": 0.2360476702451706,
|
|
"ht_mnpo/residual": 0.22854766249656677,
|
|
"ht_mnpo/residual_abs": 0.2305394411087036,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -1.8612133264541626,
|
|
"logits/rejected": -2.1879470348358154,
|
|
"logps/chosen": -0.29452234506607056,
|
|
"logps/rejected": -0.2882519066333771,
|
|
"loss": 0.5280643701553345,
|
|
"loss/core": 0.5280643701553345,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.631542205810547,
|
|
"rewards/margins": 2.3604767322540283,
|
|
"rewards/rejected": 1.271065592765808,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6780065652044166,
|
|
"grad_norm": 496.0,
|
|
"ht_mnpo/logit": 0.27381792664527893,
|
|
"ht_mnpo/residual": 0.2663179337978363,
|
|
"ht_mnpo/residual_abs": 0.2707618176937103,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.8524887561798096,
|
|
"logits/rejected": -2.209779739379883,
|
|
"logps/chosen": -0.3093031048774719,
|
|
"logps/rejected": -0.28702372312545776,
|
|
"loss": 1.0192804336547852,
|
|
"loss/core": 1.0192804336547852,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.5826401710510254,
|
|
"rewards/margins": 2.7381794452667236,
|
|
"rewards/rejected": 0.8444606065750122,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6827812593255744,
|
|
"grad_norm": 13.5,
|
|
"ht_mnpo/logit": 0.17260591685771942,
|
|
"ht_mnpo/residual": 0.1651059240102768,
|
|
"ht_mnpo/residual_abs": 0.16933459043502808,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -1.8810266256332397,
|
|
"logits/rejected": -2.1536669731140137,
|
|
"logps/chosen": -0.3034297525882721,
|
|
"logps/rejected": -0.29463475942611694,
|
|
"loss": 0.3258531093597412,
|
|
"loss/core": 0.3258531093597412,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0421574115753174,
|
|
"rewards/margins": 1.7260591983795166,
|
|
"rewards/rejected": 1.3160979747772217,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6875559534467324,
|
|
"grad_norm": 79.5,
|
|
"ht_mnpo/logit": 0.241688534617424,
|
|
"ht_mnpo/residual": 0.234188511967659,
|
|
"ht_mnpo/residual_abs": 0.23640303313732147,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -2.146207332611084,
|
|
"logits/rejected": -2.2886502742767334,
|
|
"logps/chosen": -0.309707373380661,
|
|
"logps/rejected": -0.309203565120697,
|
|
"loss": 0.458344042301178,
|
|
"loss/core": 0.458344042301178,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.303243637084961,
|
|
"rewards/margins": 2.4168851375579834,
|
|
"rewards/rejected": 0.886358380317688,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6923306475678902,
|
|
"grad_norm": 26.25,
|
|
"ht_mnpo/logit": 0.16217423975467682,
|
|
"ht_mnpo/residual": 0.15467426180839539,
|
|
"ht_mnpo/residual_abs": 0.15796169638633728,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -1.9593452215194702,
|
|
"logits/rejected": -2.2494146823883057,
|
|
"logps/chosen": -0.275308758020401,
|
|
"logps/rejected": -0.27825742959976196,
|
|
"loss": 0.21573790907859802,
|
|
"loss/core": 0.21573790907859802,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.375974178314209,
|
|
"rewards/margins": 1.621742606163025,
|
|
"rewards/rejected": 0.7542315125465393,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.697105341689048,
|
|
"grad_norm": 332.0,
|
|
"ht_mnpo/logit": 0.381821870803833,
|
|
"ht_mnpo/residual": 0.374321848154068,
|
|
"ht_mnpo/residual_abs": 0.3764488399028778,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.0782310962677,
|
|
"logits/rejected": -2.2460265159606934,
|
|
"logps/chosen": -0.31326770782470703,
|
|
"logps/rejected": -0.3076656758785248,
|
|
"loss": 1.6463673114776611,
|
|
"loss/core": 1.6463673114776611,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.482171535491943,
|
|
"rewards/margins": 3.81821870803833,
|
|
"rewards/rejected": 0.6639531254768372,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7018800358102059,
|
|
"grad_norm": 5.34375,
|
|
"ht_mnpo/logit": 0.15911749005317688,
|
|
"ht_mnpo/residual": 0.15161748230457306,
|
|
"ht_mnpo/residual_abs": 0.17043578624725342,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -1.9329124689102173,
|
|
"logits/rejected": -2.1531574726104736,
|
|
"logps/chosen": -0.32668760418891907,
|
|
"logps/rejected": -0.3379434049129486,
|
|
"loss": 0.3836316466331482,
|
|
"loss/core": 0.3836316466331482,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.7718982696533203,
|
|
"rewards/margins": 1.591174840927124,
|
|
"rewards/rejected": 1.1807234287261963,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.7066547299313638,
|
|
"grad_norm": 187.0,
|
|
"ht_mnpo/logit": 0.21543124318122864,
|
|
"ht_mnpo/residual": 0.20793123543262482,
|
|
"ht_mnpo/residual_abs": 0.21036949753761292,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -1.9197518825531006,
|
|
"logits/rejected": -2.1591908931732178,
|
|
"logps/chosen": -0.2565429210662842,
|
|
"logps/rejected": -0.2755061984062195,
|
|
"loss": 0.30916422605514526,
|
|
"loss/core": 0.30916422605514526,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.222672939300537,
|
|
"rewards/margins": 2.1543123722076416,
|
|
"rewards/rejected": 1.068360686302185,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7114294240525216,
|
|
"grad_norm": 500.0,
|
|
"ht_mnpo/logit": 0.26379165053367615,
|
|
"ht_mnpo/residual": 0.2562916874885559,
|
|
"ht_mnpo/residual_abs": 0.25983983278274536,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -1.9132686853408813,
|
|
"logits/rejected": -2.159632921218872,
|
|
"logps/chosen": -0.26915454864501953,
|
|
"logps/rejected": -0.26063108444213867,
|
|
"loss": 0.9998035430908203,
|
|
"loss/core": 0.9998035430908203,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.6523070335388184,
|
|
"rewards/margins": 2.6379168033599854,
|
|
"rewards/rejected": 1.0143901109695435,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7162041181736795,
|
|
"grad_norm": 3.765625,
|
|
"ht_mnpo/logit": 0.2811947762966156,
|
|
"ht_mnpo/residual": 0.273694783449173,
|
|
"ht_mnpo/residual_abs": 0.2760910987854004,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.0892138481140137,
|
|
"logits/rejected": -2.2944881916046143,
|
|
"logps/chosen": -0.2737082839012146,
|
|
"logps/rejected": -0.259973406791687,
|
|
"loss": 1.173396110534668,
|
|
"loss/core": 1.173396110534668,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.8202099800109863,
|
|
"rewards/margins": 2.8119475841522217,
|
|
"rewards/rejected": 1.0082619190216064,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.7209788122948374,
|
|
"grad_norm": 2.140625,
|
|
"ht_mnpo/logit": 0.2567535936832428,
|
|
"ht_mnpo/residual": 0.24925358593463898,
|
|
"ht_mnpo/residual_abs": 0.25218337774276733,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -2.019989252090454,
|
|
"logits/rejected": -2.1991806030273438,
|
|
"logps/chosen": -0.3405812978744507,
|
|
"logps/rejected": -0.31987202167510986,
|
|
"loss": 1.182205319404602,
|
|
"loss/core": 1.182205319404602,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.823350429534912,
|
|
"rewards/margins": 2.567535877227783,
|
|
"rewards/rejected": 1.255814790725708,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7257535064159952,
|
|
"grad_norm": 82.5,
|
|
"ht_mnpo/logit": 0.40269652009010315,
|
|
"ht_mnpo/residual": 0.39519649744033813,
|
|
"ht_mnpo/residual_abs": 0.39851316809654236,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -1.9100509881973267,
|
|
"logits/rejected": -2.2545628547668457,
|
|
"logps/chosen": -0.31178393959999084,
|
|
"logps/rejected": -0.28806573152542114,
|
|
"loss": 1.7591040134429932,
|
|
"loss/core": 1.7591040134429932,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 5.450934410095215,
|
|
"rewards/margins": 4.026965141296387,
|
|
"rewards/rejected": 1.4239691495895386,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7305282005371531,
|
|
"grad_norm": 5.9375,
|
|
"ht_mnpo/logit": 0.24737465381622314,
|
|
"ht_mnpo/residual": 0.23987463116645813,
|
|
"ht_mnpo/residual_abs": 0.24287545680999756,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -1.9502792358398438,
|
|
"logits/rejected": -2.0989506244659424,
|
|
"logps/chosen": -0.2815241813659668,
|
|
"logps/rejected": -0.29228392243385315,
|
|
"loss": 0.46429914236068726,
|
|
"loss/core": 0.46429914236068726,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.6968235969543457,
|
|
"rewards/margins": 2.4737465381622314,
|
|
"rewards/rejected": 1.2230769395828247,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.735302894658311,
|
|
"grad_norm": 88.0,
|
|
"ht_mnpo/logit": 0.365441232919693,
|
|
"ht_mnpo/residual": 0.357941210269928,
|
|
"ht_mnpo/residual_abs": 0.3601204752922058,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.895281195640564,
|
|
"logits/rejected": -2.2292673587799072,
|
|
"logps/chosen": -0.3453708291053772,
|
|
"logps/rejected": -0.3260127305984497,
|
|
"loss": 1.5065122842788696,
|
|
"loss/core": 1.5065122842788696,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.30310583114624,
|
|
"rewards/margins": 3.6544125080108643,
|
|
"rewards/rejected": 0.6486933827400208,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7400775887794688,
|
|
"grad_norm": 10.25,
|
|
"ht_mnpo/logit": 0.21786002814769745,
|
|
"ht_mnpo/residual": 0.21036005020141602,
|
|
"ht_mnpo/residual_abs": 0.2132169008255005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -1.9512803554534912,
|
|
"logits/rejected": -2.2073264122009277,
|
|
"logps/chosen": -0.3291786313056946,
|
|
"logps/rejected": -0.31446442008018494,
|
|
"loss": 0.46908074617385864,
|
|
"loss/core": 0.46908074617385864,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.122516393661499,
|
|
"rewards/margins": 2.178600549697876,
|
|
"rewards/rejected": 0.9439161419868469,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7448522829006267,
|
|
"grad_norm": 5.03125,
|
|
"ht_mnpo/logit": 0.2005649358034134,
|
|
"ht_mnpo/residual": 0.19306494295597076,
|
|
"ht_mnpo/residual_abs": 0.19553616642951965,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.147618293762207,
|
|
"logits/rejected": -2.329775333404541,
|
|
"logps/chosen": -0.2661969065666199,
|
|
"logps/rejected": -0.2787913680076599,
|
|
"loss": 0.28196778893470764,
|
|
"loss/core": 0.28196778893470764,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.99574613571167,
|
|
"rewards/margins": 2.0056493282318115,
|
|
"rewards/rejected": 0.9900972247123718,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7496269770217845,
|
|
"grad_norm": 5.625,
|
|
"ht_mnpo/logit": 0.23682251572608948,
|
|
"ht_mnpo/residual": 0.22932252287864685,
|
|
"ht_mnpo/residual_abs": 0.23593783378601074,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.9615819454193115,
|
|
"logits/rejected": -2.3150079250335693,
|
|
"logps/chosen": -0.3067627549171448,
|
|
"logps/rejected": -0.2893941402435303,
|
|
"loss": 0.7686271667480469,
|
|
"loss/core": 0.7686271667480469,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.3052401542663574,
|
|
"rewards/margins": 2.36822509765625,
|
|
"rewards/rejected": 0.9370150566101074,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7544016711429424,
|
|
"grad_norm": 91.5,
|
|
"ht_mnpo/logit": 0.26232343912124634,
|
|
"ht_mnpo/residual": 0.2548234462738037,
|
|
"ht_mnpo/residual_abs": 0.25668424367904663,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -1.9566657543182373,
|
|
"logits/rejected": -2.261164426803589,
|
|
"logps/chosen": -0.32755106687545776,
|
|
"logps/rejected": -0.3085654377937317,
|
|
"loss": 0.5578528642654419,
|
|
"loss/core": 0.5578528642654419,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.1903016567230225,
|
|
"rewards/margins": 2.623234272003174,
|
|
"rewards/rejected": 0.5670672059059143,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7591763652641003,
|
|
"grad_norm": 724.0,
|
|
"ht_mnpo/logit": 0.31201422214508057,
|
|
"ht_mnpo/residual": 0.30451422929763794,
|
|
"ht_mnpo/residual_abs": 0.30678969621658325,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -2.053274393081665,
|
|
"logits/rejected": -2.2694168090820312,
|
|
"logps/chosen": -0.3137114942073822,
|
|
"logps/rejected": -0.29724258184432983,
|
|
"loss": 0.9311115145683289,
|
|
"loss/core": 0.9311115145683289,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.055318832397461,
|
|
"rewards/margins": 3.1201424598693848,
|
|
"rewards/rejected": 0.9351763725280762,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7639510593852581,
|
|
"grad_norm": 23.625,
|
|
"ht_mnpo/logit": 0.20176801085472107,
|
|
"ht_mnpo/residual": 0.19426801800727844,
|
|
"ht_mnpo/residual_abs": 0.19569215178489685,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.0264482498168945,
|
|
"logits/rejected": -2.240051746368408,
|
|
"logps/chosen": -0.2773614525794983,
|
|
"logps/rejected": -0.2761766314506531,
|
|
"loss": 0.31296461820602417,
|
|
"loss/core": 0.31296461820602417,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 3.1320345401763916,
|
|
"rewards/margins": 2.0176799297332764,
|
|
"rewards/rejected": 1.1143543720245361,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.768725753506416,
|
|
"grad_norm": 338.0,
|
|
"ht_mnpo/logit": 0.31727129220962524,
|
|
"ht_mnpo/residual": 0.3097712993621826,
|
|
"ht_mnpo/residual_abs": 0.3111569285392761,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.038196086883545,
|
|
"logits/rejected": -2.4574263095855713,
|
|
"logps/chosen": -0.30098554491996765,
|
|
"logps/rejected": -0.29372772574424744,
|
|
"loss": 0.8073376417160034,
|
|
"loss/core": 0.8073376417160034,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 4.151384353637695,
|
|
"rewards/margins": 3.172712802886963,
|
|
"rewards/rejected": 0.9786712527275085,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.7735004476275739,
|
|
"grad_norm": 540.0,
|
|
"ht_mnpo/logit": 0.21397161483764648,
|
|
"ht_mnpo/residual": 0.20647163689136505,
|
|
"ht_mnpo/residual_abs": 0.21008792519569397,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -1.9568876028060913,
|
|
"logits/rejected": -2.162205457687378,
|
|
"logps/chosen": -0.32081520557403564,
|
|
"logps/rejected": -0.28566688299179077,
|
|
"loss": 0.5576066970825195,
|
|
"loss/core": 0.5576066970825195,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.346921443939209,
|
|
"rewards/margins": 2.139716386795044,
|
|
"rewards/rejected": 1.2072051763534546,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7782751417487317,
|
|
"grad_norm": 120.5,
|
|
"ht_mnpo/logit": 0.34778210520744324,
|
|
"ht_mnpo/residual": 0.34028205275535583,
|
|
"ht_mnpo/residual_abs": 0.34224414825439453,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -1.81075119972229,
|
|
"logits/rejected": -2.0091278553009033,
|
|
"logps/chosen": -0.28327834606170654,
|
|
"logps/rejected": -0.2862473726272583,
|
|
"loss": 0.8907841444015503,
|
|
"loss/core": 0.8907841444015503,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.21142578125,
|
|
"rewards/margins": 3.477820873260498,
|
|
"rewards/rejected": 0.7336045503616333,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7830498358698896,
|
|
"grad_norm": 14.1875,
|
|
"ht_mnpo/logit": 0.15176530182361603,
|
|
"ht_mnpo/residual": 0.1442653089761734,
|
|
"ht_mnpo/residual_abs": 0.14901027083396912,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -1.9298378229141235,
|
|
"logits/rejected": -2.1080105304718018,
|
|
"logps/chosen": -0.2902945578098297,
|
|
"logps/rejected": -0.309215247631073,
|
|
"loss": 0.4051854610443115,
|
|
"loss/core": 0.4051854610443115,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.702075481414795,
|
|
"rewards/margins": 1.517652988433838,
|
|
"rewards/rejected": 1.184422492980957,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.7878245299910475,
|
|
"grad_norm": 1312.0,
|
|
"ht_mnpo/logit": 0.3178713321685791,
|
|
"ht_mnpo/residual": 0.3103713393211365,
|
|
"ht_mnpo/residual_abs": 0.3130297064781189,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.0223171710968018,
|
|
"logits/rejected": -2.2735869884490967,
|
|
"logps/chosen": -0.34258824586868286,
|
|
"logps/rejected": -0.2813429832458496,
|
|
"loss": 0.8753134608268738,
|
|
"loss/core": 0.8753134608268738,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.9592788219451904,
|
|
"rewards/margins": 3.178713321685791,
|
|
"rewards/rejected": 0.7805658578872681,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7925992241122053,
|
|
"grad_norm": 29.625,
|
|
"ht_mnpo/logit": 0.232127383351326,
|
|
"ht_mnpo/residual": 0.22462734580039978,
|
|
"ht_mnpo/residual_abs": 0.22618567943572998,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.1135377883911133,
|
|
"logits/rejected": -2.3391060829162598,
|
|
"logps/chosen": -0.27336445450782776,
|
|
"logps/rejected": -0.28452619910240173,
|
|
"loss": 0.5279203653335571,
|
|
"loss/core": 0.5279203653335571,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.090513229370117,
|
|
"rewards/margins": 2.3212738037109375,
|
|
"rewards/rejected": 0.7692393660545349,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7973739182333631,
|
|
"grad_norm": 31.75,
|
|
"ht_mnpo/logit": 0.23250722885131836,
|
|
"ht_mnpo/residual": 0.22500717639923096,
|
|
"ht_mnpo/residual_abs": 0.22754648327827454,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -1.7653303146362305,
|
|
"logits/rejected": -1.9559898376464844,
|
|
"logps/chosen": -0.2960887551307678,
|
|
"logps/rejected": -0.3074343502521515,
|
|
"loss": 0.5747458934783936,
|
|
"loss/core": 0.5747458934783936,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.5430405139923096,
|
|
"rewards/margins": 2.3250718116760254,
|
|
"rewards/rejected": 1.217968463897705,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8021486123545211,
|
|
"grad_norm": 434.0,
|
|
"ht_mnpo/logit": 0.22585304081439972,
|
|
"ht_mnpo/residual": 0.2183530032634735,
|
|
"ht_mnpo/residual_abs": 0.2201291024684906,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.053584337234497,
|
|
"logits/rejected": -2.245476722717285,
|
|
"logps/chosen": -0.3269621729850769,
|
|
"logps/rejected": -0.26414385437965393,
|
|
"loss": 0.5430350303649902,
|
|
"loss/core": 0.5430350303649902,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.252704620361328,
|
|
"rewards/margins": 2.2585301399230957,
|
|
"rewards/rejected": 0.9941746592521667,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8069233064756789,
|
|
"grad_norm": 112.5,
|
|
"ht_mnpo/logit": 0.2180916965007782,
|
|
"ht_mnpo/residual": 0.21059170365333557,
|
|
"ht_mnpo/residual_abs": 0.21333935856819153,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -1.9658924341201782,
|
|
"logits/rejected": -2.281589984893799,
|
|
"logps/chosen": -0.30036720633506775,
|
|
"logps/rejected": -0.2967119812965393,
|
|
"loss": 0.3782276511192322,
|
|
"loss/core": 0.3782276511192322,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.758944511413574,
|
|
"rewards/margins": 2.1809170246124268,
|
|
"rewards/rejected": 0.5780273079872131,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8116980005968367,
|
|
"grad_norm": 2.140625,
|
|
"ht_mnpo/logit": 0.2448008507490158,
|
|
"ht_mnpo/residual": 0.23730087280273438,
|
|
"ht_mnpo/residual_abs": 0.23997625708580017,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -1.8780618906021118,
|
|
"logits/rejected": -2.181715488433838,
|
|
"logps/chosen": -0.2794502377510071,
|
|
"logps/rejected": -0.26828446984291077,
|
|
"loss": 0.8545548319816589,
|
|
"loss/core": 0.8545548319816589,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.7230312824249268,
|
|
"rewards/margins": 2.4480087757110596,
|
|
"rewards/rejected": 1.2750225067138672,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8164726947179947,
|
|
"grad_norm": 11.0625,
|
|
"ht_mnpo/logit": 0.1709764301776886,
|
|
"ht_mnpo/residual": 0.16347643733024597,
|
|
"ht_mnpo/residual_abs": 0.1738419234752655,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -2.113243579864502,
|
|
"logits/rejected": -2.4351394176483154,
|
|
"logps/chosen": -0.26266947388648987,
|
|
"logps/rejected": -0.2733474373817444,
|
|
"loss": 0.12632545828819275,
|
|
"loss/core": 0.12632545828819275,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.9499731063842773,
|
|
"rewards/margins": 1.7097641229629517,
|
|
"rewards/rejected": 1.2402087450027466,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8212473888391525,
|
|
"grad_norm": 3.296875,
|
|
"ht_mnpo/logit": 0.14397737383842468,
|
|
"ht_mnpo/residual": 0.13647738099098206,
|
|
"ht_mnpo/residual_abs": 0.13811519742012024,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.9785963296890259,
|
|
"logits/rejected": -2.251053810119629,
|
|
"logps/chosen": -0.27199432253837585,
|
|
"logps/rejected": -0.27827268838882446,
|
|
"loss": 0.15808339416980743,
|
|
"loss/core": 0.15808339416980743,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.0167627334594727,
|
|
"rewards/margins": 1.4397737979888916,
|
|
"rewards/rejected": 0.5769888758659363,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.8260220829603103,
|
|
"grad_norm": 161.0,
|
|
"ht_mnpo/logit": 0.3069954812526703,
|
|
"ht_mnpo/residual": 0.2994953989982605,
|
|
"ht_mnpo/residual_abs": 0.3049822747707367,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.8511345386505127,
|
|
"logits/rejected": -2.2174289226531982,
|
|
"logps/chosen": -0.3130944073200226,
|
|
"logps/rejected": -0.3080425560474396,
|
|
"loss": 1.1950414180755615,
|
|
"loss/core": 1.1950414180755615,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.894388198852539,
|
|
"rewards/margins": 3.0699546337127686,
|
|
"rewards/rejected": 0.8244336247444153,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8307967770814683,
|
|
"grad_norm": 29.75,
|
|
"ht_mnpo/logit": 0.09861830621957779,
|
|
"ht_mnpo/residual": 0.09111831337213516,
|
|
"ht_mnpo/residual_abs": 0.09437834471464157,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.1397294998168945,
|
|
"logits/rejected": -2.3984568119049072,
|
|
"logps/chosen": -0.28324785828590393,
|
|
"logps/rejected": -0.28805121779441833,
|
|
"loss": 0.06719032675027847,
|
|
"loss/core": 0.06719032675027847,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.770001769065857,
|
|
"rewards/margins": 0.9861831665039062,
|
|
"rewards/rejected": 0.7838188409805298,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8355714712026261,
|
|
"grad_norm": 3.078125,
|
|
"ht_mnpo/logit": 0.11863557994365692,
|
|
"ht_mnpo/residual": 0.1111355796456337,
|
|
"ht_mnpo/residual_abs": 0.11609260737895966,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -1.9720087051391602,
|
|
"logits/rejected": -2.288149356842041,
|
|
"logps/chosen": -0.2549072802066803,
|
|
"logps/rejected": -0.27585774660110474,
|
|
"loss": 0.08893564343452454,
|
|
"loss/core": 0.08893564343452454,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.8031947612762451,
|
|
"rewards/margins": 1.1863558292388916,
|
|
"rewards/rejected": 0.6168391108512878,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.8403461653237839,
|
|
"grad_norm": 50.75,
|
|
"ht_mnpo/logit": 0.13304957747459412,
|
|
"ht_mnpo/residual": 0.1255495846271515,
|
|
"ht_mnpo/residual_abs": 0.12660455703735352,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.260289430618286,
|
|
"logits/rejected": -2.4413468837738037,
|
|
"logps/chosen": -0.304781973361969,
|
|
"logps/rejected": -0.30884766578674316,
|
|
"loss": 0.14404062926769257,
|
|
"loss/core": 0.14404062926769257,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 1.8423153162002563,
|
|
"rewards/margins": 1.3304959535598755,
|
|
"rewards/rejected": 0.5118193626403809,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8451208594449419,
|
|
"grad_norm": 2.796875,
|
|
"ht_mnpo/logit": 0.2581387162208557,
|
|
"ht_mnpo/residual": 0.2506386935710907,
|
|
"ht_mnpo/residual_abs": 0.2524881660938263,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -1.9322468042373657,
|
|
"logits/rejected": -2.3333423137664795,
|
|
"logps/chosen": -0.319403737783432,
|
|
"logps/rejected": -0.3051341474056244,
|
|
"loss": 0.6756513714790344,
|
|
"loss/core": 0.6756513714790344,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.50068736076355,
|
|
"rewards/margins": 2.5813872814178467,
|
|
"rewards/rejected": 0.9193001985549927,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8498955535660997,
|
|
"grad_norm": 12.8125,
|
|
"ht_mnpo/logit": 0.22012737393379211,
|
|
"ht_mnpo/residual": 0.2126273661851883,
|
|
"ht_mnpo/residual_abs": 0.2149299830198288,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.9026530981063843,
|
|
"logits/rejected": -2.1633448600769043,
|
|
"logps/chosen": -0.29086828231811523,
|
|
"logps/rejected": -0.27807530760765076,
|
|
"loss": 0.35099947452545166,
|
|
"loss/core": 0.35099947452545166,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.9746203422546387,
|
|
"rewards/margins": 2.2012736797332764,
|
|
"rewards/rejected": 0.7733467817306519,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.8546702476872575,
|
|
"grad_norm": 15.8125,
|
|
"ht_mnpo/logit": 0.3148799538612366,
|
|
"ht_mnpo/residual": 0.3073800206184387,
|
|
"ht_mnpo/residual_abs": 0.3098318874835968,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -1.93404221534729,
|
|
"logits/rejected": -2.2042999267578125,
|
|
"logps/chosen": -0.32174813747406006,
|
|
"logps/rejected": -0.2906070351600647,
|
|
"loss": 0.975996196269989,
|
|
"loss/core": 0.975996196269989,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.408227443695068,
|
|
"rewards/margins": 3.1487996578216553,
|
|
"rewards/rejected": 1.259427785873413,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8594449418084154,
|
|
"grad_norm": 16.5,
|
|
"ht_mnpo/logit": 0.22211775183677673,
|
|
"ht_mnpo/residual": 0.2146177738904953,
|
|
"ht_mnpo/residual_abs": 0.26752087473869324,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -1.908017873764038,
|
|
"logits/rejected": -2.071946382522583,
|
|
"logps/chosen": -0.30949825048446655,
|
|
"logps/rejected": -0.33156439661979675,
|
|
"loss": 0.5480181574821472,
|
|
"loss/core": 0.5480181574821472,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.108867645263672,
|
|
"rewards/margins": 2.221177577972412,
|
|
"rewards/rejected": 1.8876899480819702,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8594449418084154,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.7468786084651947,
|
|
"train_runtime": 6934.2752,
|
|
"train_samples_per_second": 2.077,
|
|
"train_steps_per_second": 0.13
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|