4544 lines
178 KiB
JSON
4544 lines
178 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8589835361488904,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.22921419143676758,
|
|
"drift/rejected_abs": 0.10946719348430634,
|
|
"epoch": 0.004772130756382725,
|
|
"grad_norm": 8.6875,
|
|
"ht_mnpo/logit": 0.12133613973855972,
|
|
"ht_mnpo/residual": 0.1163361445069313,
|
|
"ht_mnpo/residual_abs": 0.15141648054122925,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.111111111111111e-08,
|
|
"logits/chosen": -2.3204128742218018,
|
|
"logits/rejected": -2.2550835609436035,
|
|
"logps/chosen": -0.29364365339279175,
|
|
"logps/rejected": -0.2911434769630432,
|
|
"loss": 0.16518738865852356,
|
|
"loss/core": 0.1614948809146881,
|
|
"loss/preference_sft": 0.29364365339279175,
|
|
"loss/reference_anchor": 0.15526053309440613,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.292141914367676,
|
|
"rewards/margins": 1.2133615016937256,
|
|
"rewards/rejected": 1.0787802934646606,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3288218379020691,
|
|
"drift/rejected_abs": 0.1325603723526001,
|
|
"epoch": 0.00954426151276545,
|
|
"grad_norm": 892.0,
|
|
"ht_mnpo/logit": 0.19652661681175232,
|
|
"ht_mnpo/residual": 0.1915266215801239,
|
|
"ht_mnpo/residual_abs": 0.20962563157081604,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.5e-08,
|
|
"logits/chosen": -2.04286527633667,
|
|
"logits/rejected": -1.97250235080719,
|
|
"logps/chosen": -0.27547430992126465,
|
|
"logps/rejected": -0.2841722369194031,
|
|
"loss": 0.6829333901405334,
|
|
"loss/core": 0.6688618659973145,
|
|
"loss/preference_sft": 0.27547430992126465,
|
|
"loss/reference_anchor": 0.6760290861129761,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.287240982055664,
|
|
"rewards/margins": 1.9652659893035889,
|
|
"rewards/rejected": 1.321974754333496,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3020998537540436,
|
|
"drift/rejected_abs": 0.08261941373348236,
|
|
"epoch": 0.014316392269148175,
|
|
"grad_norm": 2.203125,
|
|
"ht_mnpo/logit": 0.2266816794872284,
|
|
"ht_mnpo/residual": 0.22168168425559998,
|
|
"ht_mnpo/residual_abs": 0.23883593082427979,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.888888888888889e-08,
|
|
"logits/chosen": -2.175687551498413,
|
|
"logits/rejected": -2.1948294639587402,
|
|
"logps/chosen": -0.3126169443130493,
|
|
"logps/rejected": -0.3139331340789795,
|
|
"loss": 0.7264648079872131,
|
|
"loss/core": 0.7176737189292908,
|
|
"loss/preference_sft": 0.3126169443130493,
|
|
"loss/reference_anchor": 0.4082910120487213,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0202560424804688,
|
|
"rewards/margins": 2.2668166160583496,
|
|
"rewards/rejected": 0.7534393072128296,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28461581468582153,
|
|
"drift/rejected_abs": 0.1129002794623375,
|
|
"epoch": 0.0190885230255309,
|
|
"grad_norm": 174.0,
|
|
"ht_mnpo/logit": 0.1719159036874771,
|
|
"ht_mnpo/residual": 0.1669158935546875,
|
|
"ht_mnpo/residual_abs": 0.18884992599487305,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.2777777777777776e-08,
|
|
"logits/chosen": -2.1989941596984863,
|
|
"logits/rejected": -2.175231456756592,
|
|
"logps/chosen": -0.31303125619888306,
|
|
"logps/rejected": -0.3244328498840332,
|
|
"loss": 0.6520941853523254,
|
|
"loss/core": 0.6402677297592163,
|
|
"loss/preference_sft": 0.31303125619888306,
|
|
"loss/reference_anchor": 0.5600210428237915,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.845700740814209,
|
|
"rewards/margins": 1.7191591262817383,
|
|
"rewards/rejected": 1.1265417337417603,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2910028398036957,
|
|
"drift/rejected_abs": 0.1626802384853363,
|
|
"epoch": 0.023860653781913623,
|
|
"grad_norm": 1.5625,
|
|
"ht_mnpo/logit": 0.12845146656036377,
|
|
"ht_mnpo/residual": 0.12345147132873535,
|
|
"ht_mnpo/residual_abs": 0.14755485951900482,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.666666666666667e-08,
|
|
"logits/chosen": -2.1143784523010254,
|
|
"logits/rejected": -2.0981764793395996,
|
|
"logps/chosen": -0.3009456694126129,
|
|
"logps/rejected": -0.2978978455066681,
|
|
"loss": 0.28243863582611084,
|
|
"loss/core": 0.2706168591976166,
|
|
"loss/preference_sft": 0.3009456694126129,
|
|
"loss/reference_anchor": 0.5609945058822632,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.9088892936706543,
|
|
"rewards/margins": 1.2845146656036377,
|
|
"rewards/rejected": 1.6243746280670166,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.47175097465515137,
|
|
"drift/rejected_abs": 0.252032995223999,
|
|
"epoch": 0.02863278453829635,
|
|
"grad_norm": 13.1875,
|
|
"ht_mnpo/logit": 0.2227689027786255,
|
|
"ht_mnpo/residual": 0.21776887774467468,
|
|
"ht_mnpo/residual_abs": 0.35318318009376526,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.055555555555555e-08,
|
|
"logits/chosen": -1.7267528772354126,
|
|
"logits/rejected": -1.8169645071029663,
|
|
"logps/chosen": -0.2687901258468628,
|
|
"logps/rejected": -0.24951688945293427,
|
|
"loss": 1.0632941722869873,
|
|
"loss/core": 1.041975498199463,
|
|
"loss/preference_sft": 0.2687901258468628,
|
|
"loss/reference_anchor": 1.039062738418579,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.71492862701416,
|
|
"rewards/margins": 2.227689266204834,
|
|
"rewards/rejected": 2.4872403144836426,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17663131654262543,
|
|
"drift/rejected_abs": 0.24685576558113098,
|
|
"epoch": 0.03340491529467907,
|
|
"grad_norm": 147.0,
|
|
"ht_mnpo/logit": -0.069961316883564,
|
|
"ht_mnpo/residual": -0.07496131211519241,
|
|
"ht_mnpo/residual_abs": 0.25241637229919434,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.444444444444444e-08,
|
|
"logits/chosen": -2.023242712020874,
|
|
"logits/rejected": -1.947217345237732,
|
|
"logps/chosen": -0.30046090483665466,
|
|
"logps/rejected": -0.2926713526248932,
|
|
"loss": 0.7559306025505066,
|
|
"loss/core": 0.745723307132721,
|
|
"loss/preference_sft": 0.30046090483665466,
|
|
"loss/reference_anchor": 0.4803217053413391,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7663133144378662,
|
|
"rewards/margins": -0.6996129751205444,
|
|
"rewards/rejected": 2.4659264087677,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2406160533428192,
|
|
"drift/rejected_abs": 0.17416922748088837,
|
|
"epoch": 0.0381770460510618,
|
|
"grad_norm": 22.625,
|
|
"ht_mnpo/logit": 0.06698594987392426,
|
|
"ht_mnpo/residual": 0.06198593229055405,
|
|
"ht_mnpo/residual_abs": 0.2546267509460449,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.0833333333333334e-07,
|
|
"logits/chosen": -2.0492732524871826,
|
|
"logits/rejected": -2.0772995948791504,
|
|
"logps/chosen": -0.28532397747039795,
|
|
"logps/rejected": -0.3091915547847748,
|
|
"loss": 0.7844445109367371,
|
|
"loss/core": 0.773983895778656,
|
|
"loss/preference_sft": 0.28532397747039795,
|
|
"loss/reference_anchor": 0.49450117349624634,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.406160831451416,
|
|
"rewards/margins": 0.6698596477508545,
|
|
"rewards/rejected": 1.736301064491272,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23510071635246277,
|
|
"drift/rejected_abs": 0.14974211156368256,
|
|
"epoch": 0.04294917680744453,
|
|
"grad_norm": 14.125,
|
|
"ht_mnpo/logit": 0.08539210259914398,
|
|
"ht_mnpo/residual": 0.08039210736751556,
|
|
"ht_mnpo/residual_abs": 0.14728103578090668,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.2222222222222222e-07,
|
|
"logits/chosen": -1.996901512145996,
|
|
"logits/rejected": -1.968118667602539,
|
|
"logps/chosen": -0.2885960042476654,
|
|
"logps/rejected": -0.28996142745018005,
|
|
"loss": 0.1255718469619751,
|
|
"loss/core": 0.12194611132144928,
|
|
"loss/preference_sft": 0.2885960042476654,
|
|
"loss/reference_anchor": 0.15242737531661987,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3506722450256348,
|
|
"rewards/margins": 0.853921115398407,
|
|
"rewards/rejected": 1.496751308441162,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28709280490875244,
|
|
"drift/rejected_abs": 0.0940680131316185,
|
|
"epoch": 0.047721307563827246,
|
|
"grad_norm": 43.0,
|
|
"ht_mnpo/logit": 0.19577643275260925,
|
|
"ht_mnpo/residual": 0.19077643752098083,
|
|
"ht_mnpo/residual_abs": 0.19558581709861755,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3611111111111108e-07,
|
|
"logits/chosen": -2.2561166286468506,
|
|
"logits/rejected": -2.199009895324707,
|
|
"logps/chosen": -0.271379679441452,
|
|
"logps/rejected": -0.2756428122520447,
|
|
"loss": 0.4556834101676941,
|
|
"loss/core": 0.44865885376930237,
|
|
"loss/preference_sft": 0.271379679441452,
|
|
"loss/reference_anchor": 0.3240897059440613,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.870856761932373,
|
|
"rewards/margins": 1.9577643871307373,
|
|
"rewards/rejected": 0.9130924940109253,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22801081836223602,
|
|
"drift/rejected_abs": 0.13098636269569397,
|
|
"epoch": 0.05249343832020997,
|
|
"grad_norm": 14.6875,
|
|
"ht_mnpo/logit": 0.09714122861623764,
|
|
"ht_mnpo/residual": 0.09214121848344803,
|
|
"ht_mnpo/residual_abs": 0.10519029945135117,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.5e-07,
|
|
"logits/chosen": -2.0955872535705566,
|
|
"logits/rejected": -2.0516104698181152,
|
|
"logps/chosen": -0.28714004158973694,
|
|
"logps/rejected": -0.28851160407066345,
|
|
"loss": 0.11324092000722885,
|
|
"loss/core": 0.10730179399251938,
|
|
"loss/preference_sft": 0.28714004158973694,
|
|
"loss/reference_anchor": 0.2682426869869232,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.280061721801758,
|
|
"rewards/margins": 0.9714123010635376,
|
|
"rewards/rejected": 1.3086493015289307,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2036731243133545,
|
|
"drift/rejected_abs": 0.25291842222213745,
|
|
"epoch": 0.0572655690765927,
|
|
"grad_norm": 608.0,
|
|
"ht_mnpo/logit": -0.04795291647315025,
|
|
"ht_mnpo/residual": -0.05295292288064957,
|
|
"ht_mnpo/residual_abs": 0.25708532333374023,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.6388888888888888e-07,
|
|
"logits/chosen": -2.219554901123047,
|
|
"logits/rejected": -2.147869348526001,
|
|
"logps/chosen": -0.30570459365844727,
|
|
"logps/rejected": -0.30410706996917725,
|
|
"loss": 0.8895317912101746,
|
|
"loss/core": 0.877190887928009,
|
|
"loss/preference_sft": 0.30570459365844727,
|
|
"loss/reference_anchor": 0.5864753127098083,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.036237955093384,
|
|
"rewards/margins": -0.47952914237976074,
|
|
"rewards/rejected": 2.5157668590545654,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22737295925617218,
|
|
"drift/rejected_abs": 0.12715181708335876,
|
|
"epoch": 0.062037699832975426,
|
|
"grad_norm": 2.8125,
|
|
"ht_mnpo/logit": 0.10005078464746475,
|
|
"ht_mnpo/residual": 0.09505076706409454,
|
|
"ht_mnpo/residual_abs": 0.13454630970954895,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7777777777777776e-07,
|
|
"logits/chosen": -2.095898389816284,
|
|
"logits/rejected": -2.160431385040283,
|
|
"logps/chosen": -0.30385154485702515,
|
|
"logps/rejected": -0.2779307961463928,
|
|
"loss": 0.1357956975698471,
|
|
"loss/core": 0.13056448101997375,
|
|
"loss/preference_sft": 0.30385154485702515,
|
|
"loss/reference_anchor": 0.23117490112781525,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.271531581878662,
|
|
"rewards/margins": 1.0005077123641968,
|
|
"rewards/rejected": 1.2710237503051758,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1965130865573883,
|
|
"drift/rejected_abs": 0.19766323268413544,
|
|
"epoch": 0.06680983058935815,
|
|
"grad_norm": 50.25,
|
|
"ht_mnpo/logit": 0.0027785298880189657,
|
|
"ht_mnpo/residual": -0.0022214693017303944,
|
|
"ht_mnpo/residual_abs": 0.1599881798028946,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.9166666666666668e-07,
|
|
"logits/chosen": -2.147693395614624,
|
|
"logits/rejected": -2.035130739212036,
|
|
"logps/chosen": -0.28254345059394836,
|
|
"logps/rejected": -0.28443124890327454,
|
|
"loss": 0.12861734628677368,
|
|
"loss/core": 0.1251140534877777,
|
|
"loss/preference_sft": 0.28254345059394836,
|
|
"loss/reference_anchor": 0.14691027998924255,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.9649995565414429,
|
|
"rewards/margins": 0.027785325422883034,
|
|
"rewards/rejected": 1.9372141361236572,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31567615270614624,
|
|
"drift/rejected_abs": 0.1452103555202484,
|
|
"epoch": 0.07158196134574088,
|
|
"grad_norm": 808.0,
|
|
"ht_mnpo/logit": 0.17037446796894073,
|
|
"ht_mnpo/residual": 0.16537444293498993,
|
|
"ht_mnpo/residual_abs": 0.20655925571918488,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0555555555555553e-07,
|
|
"logits/chosen": -2.248660087585449,
|
|
"logits/rejected": -2.147256374359131,
|
|
"logps/chosen": -0.2782396078109741,
|
|
"logps/rejected": -0.29824963212013245,
|
|
"loss": 0.590999960899353,
|
|
"loss/core": 0.5754934549331665,
|
|
"loss/preference_sft": 0.2782396078109741,
|
|
"loss/reference_anchor": 0.7474981546401978,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.155601978302002,
|
|
"rewards/margins": 1.7037445306777954,
|
|
"rewards/rejected": 1.4518576860427856,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2219829261302948,
|
|
"drift/rejected_abs": 0.08889742195606232,
|
|
"epoch": 0.0763540921021236,
|
|
"grad_norm": 3.359375,
|
|
"ht_mnpo/logit": 0.13296504318714142,
|
|
"ht_mnpo/residual": 0.127965047955513,
|
|
"ht_mnpo/residual_abs": 0.14647246897220612,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.1944444444444442e-07,
|
|
"logits/chosen": -2.1656582355499268,
|
|
"logits/rejected": -2.274348735809326,
|
|
"logps/chosen": -0.2902677357196808,
|
|
"logps/rejected": -0.2846531271934509,
|
|
"loss": 0.25446221232414246,
|
|
"loss/core": 0.24999630451202393,
|
|
"loss/preference_sft": 0.2902677357196808,
|
|
"loss/reference_anchor": 0.19426903128623962,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.2158241271972656,
|
|
"rewards/margins": 1.3296505212783813,
|
|
"rewards/rejected": 0.8861739039421082,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19277288019657135,
|
|
"drift/rejected_abs": 0.09097670018672943,
|
|
"epoch": 0.08112622285850632,
|
|
"grad_norm": 0.376953125,
|
|
"ht_mnpo/logit": 0.10174652189016342,
|
|
"ht_mnpo/residual": 0.0967465192079544,
|
|
"ht_mnpo/residual_abs": 0.10738085210323334,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3333333333333333e-07,
|
|
"logits/chosen": -2.0793979167938232,
|
|
"logits/rejected": -2.14670991897583,
|
|
"logps/chosen": -0.30412206053733826,
|
|
"logps/rejected": -0.2832191586494446,
|
|
"loss": 0.0978093072772026,
|
|
"loss/core": 0.09453041851520538,
|
|
"loss/preference_sft": 0.30412206053733826,
|
|
"loss/reference_anchor": 0.13353168964385986,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9227644205093384,
|
|
"rewards/margins": 1.0174652338027954,
|
|
"rewards/rejected": 0.9052993655204773,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22163239121437073,
|
|
"drift/rejected_abs": 0.12397237122058868,
|
|
"epoch": 0.08589835361488905,
|
|
"grad_norm": 64.5,
|
|
"ht_mnpo/logit": 0.09769080579280853,
|
|
"ht_mnpo/residual": 0.09269081801176071,
|
|
"ht_mnpo/residual_abs": 0.1690768450498581,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4722222222222224e-07,
|
|
"logits/chosen": -2.1062588691711426,
|
|
"logits/rejected": -2.0609793663024902,
|
|
"logps/chosen": -0.3209324777126312,
|
|
"logps/rejected": -0.2916238605976105,
|
|
"loss": 0.5165913701057434,
|
|
"loss/core": 0.5093555450439453,
|
|
"loss/preference_sft": 0.3209324777126312,
|
|
"loss/reference_anchor": 0.329694539308548,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2147328853607178,
|
|
"rewards/margins": 0.9769083261489868,
|
|
"rewards/rejected": 1.237824559211731,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18284250795841217,
|
|
"drift/rejected_abs": 0.15377461910247803,
|
|
"epoch": 0.09067048437127177,
|
|
"grad_norm": 608.0,
|
|
"ht_mnpo/logit": 0.029158195480704308,
|
|
"ht_mnpo/residual": 0.02415819838643074,
|
|
"ht_mnpo/residual_abs": 0.14915595948696136,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4998495746616845e-07,
|
|
"logits/chosen": -2.258273124694824,
|
|
"logits/rejected": -2.348480701446533,
|
|
"logps/chosen": -0.3251148760318756,
|
|
"logps/rejected": -0.3082412779331207,
|
|
"loss": 0.39904260635375977,
|
|
"loss/core": 0.38921934366226196,
|
|
"loss/preference_sft": 0.3251148760318756,
|
|
"loss/reference_anchor": 0.4586527943611145,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8256721496582031,
|
|
"rewards/margins": 0.2915820777416229,
|
|
"rewards/rejected": 1.534090280532837,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39025211334228516,
|
|
"drift/rejected_abs": 0.11058616638183594,
|
|
"epoch": 0.09544261512765449,
|
|
"grad_norm": 209.0,
|
|
"ht_mnpo/logit": 0.2797187268733978,
|
|
"ht_mnpo/residual": 0.2747187614440918,
|
|
"ht_mnpo/residual_abs": 0.27753618359565735,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4992385337738696e-07,
|
|
"logits/chosen": -1.7920671701431274,
|
|
"logits/rejected": -1.848049521446228,
|
|
"logps/chosen": -0.30774667859077454,
|
|
"logps/rejected": -0.2939962148666382,
|
|
"loss": 1.4822406768798828,
|
|
"loss/core": 1.461856722831726,
|
|
"loss/preference_sft": 0.30774667859077454,
|
|
"loss/reference_anchor": 0.9884130358695984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.900453567504883,
|
|
"rewards/margins": 2.797187566757202,
|
|
"rewards/rejected": 1.1032661199569702,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2572689652442932,
|
|
"drift/rejected_abs": 0.16230344772338867,
|
|
"epoch": 0.10021474588403723,
|
|
"grad_norm": 0.498046875,
|
|
"ht_mnpo/logit": 0.09508073329925537,
|
|
"ht_mnpo/residual": 0.09008073061704636,
|
|
"ht_mnpo/residual_abs": 0.2105640172958374,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4981577053636144e-07,
|
|
"logits/chosen": -2.1005120277404785,
|
|
"logits/rejected": -2.1056771278381348,
|
|
"logps/chosen": -0.30666694045066833,
|
|
"logps/rejected": -0.3211822807788849,
|
|
"loss": 0.569188117980957,
|
|
"loss/core": 0.5602066516876221,
|
|
"loss/preference_sft": 0.30666694045066833,
|
|
"loss/reference_anchor": 0.41840967535972595,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.5712063312530518,
|
|
"rewards/margins": 0.9508073925971985,
|
|
"rewards/rejected": 1.6203992366790771,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23388180136680603,
|
|
"drift/rejected_abs": 0.10054342448711395,
|
|
"epoch": 0.10498687664041995,
|
|
"grad_norm": 0.72265625,
|
|
"ht_mnpo/logit": 0.13339105248451233,
|
|
"ht_mnpo/residual": 0.12839102745056152,
|
|
"ht_mnpo/residual_abs": 0.1422457993030548,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.496607495886281e-07,
|
|
"logits/chosen": -2.2283363342285156,
|
|
"logits/rejected": -2.157383680343628,
|
|
"logps/chosen": -0.301920086145401,
|
|
"logps/rejected": -0.31350260972976685,
|
|
"loss": 0.3251662850379944,
|
|
"loss/core": 0.3147370219230652,
|
|
"loss/preference_sft": 0.301920086145401,
|
|
"loss/reference_anchor": 0.4912687838077545,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.338407039642334,
|
|
"rewards/margins": 1.3339102268218994,
|
|
"rewards/rejected": 1.0044968128204346,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14543868601322174,
|
|
"drift/rejected_abs": 0.08579744398593903,
|
|
"epoch": 0.10975900739680267,
|
|
"grad_norm": 4.03125,
|
|
"ht_mnpo/logit": 0.05967400223016739,
|
|
"ht_mnpo/residual": 0.05467400699853897,
|
|
"ht_mnpo/residual_abs": 0.07688696682453156,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4945884883122553e-07,
|
|
"logits/chosen": -2.228116035461426,
|
|
"logits/rejected": -2.1459319591522217,
|
|
"logps/chosen": -0.2716502547264099,
|
|
"logps/rejected": -0.26580381393432617,
|
|
"loss": 0.05459178611636162,
|
|
"loss/core": 0.05268437787890434,
|
|
"loss/preference_sft": 0.2716502547264099,
|
|
"loss/reference_anchor": 0.06820501387119293,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.4522254467010498,
|
|
"rewards/margins": 0.5967400074005127,
|
|
"rewards/rejected": 0.8554852604866028,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22194862365722656,
|
|
"drift/rejected_abs": 0.1216927170753479,
|
|
"epoch": 0.1145311381531854,
|
|
"grad_norm": 23.0,
|
|
"ht_mnpo/logit": 0.10122959315776825,
|
|
"ht_mnpo/residual": 0.09622960537672043,
|
|
"ht_mnpo/residual_abs": 0.12579123675823212,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.492101441907714e-07,
|
|
"logits/chosen": -1.9909662008285522,
|
|
"logits/rejected": -2.0008864402770996,
|
|
"logps/chosen": -0.26980477571487427,
|
|
"logps/rejected": -0.26790764927864075,
|
|
"loss": 0.08855850249528885,
|
|
"loss/core": 0.08529040962457657,
|
|
"loss/preference_sft": 0.26980477571487427,
|
|
"loss/reference_anchor": 0.13642403483390808,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2187845706939697,
|
|
"rewards/margins": 1.0122959613800049,
|
|
"rewards/rejected": 1.2064883708953857,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3175174295902252,
|
|
"drift/rejected_abs": 0.18678590655326843,
|
|
"epoch": 0.11930326890956812,
|
|
"grad_norm": 768.0,
|
|
"ht_mnpo/logit": 0.13146166503429413,
|
|
"ht_mnpo/residual": 0.12646165490150452,
|
|
"ht_mnpo/residual_abs": 0.18497243523597717,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4891472919490977e-07,
|
|
"logits/chosen": -1.972855567932129,
|
|
"logits/rejected": -2.0070338249206543,
|
|
"logps/chosen": -0.3044547140598297,
|
|
"logps/rejected": -0.30631768703460693,
|
|
"loss": 0.5623260736465454,
|
|
"loss/core": 0.5474299192428589,
|
|
"loss/preference_sft": 0.3044547140598297,
|
|
"loss/reference_anchor": 0.7143507599830627,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.170875072479248,
|
|
"rewards/margins": 1.3146165609359741,
|
|
"rewards/rejected": 1.8562583923339844,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20063884556293488,
|
|
"drift/rejected_abs": 0.10184315592050552,
|
|
"epoch": 0.12407539966595085,
|
|
"grad_norm": 2.671875,
|
|
"ht_mnpo/logit": 0.11115453392267227,
|
|
"ht_mnpo/residual": 0.10615452378988266,
|
|
"ht_mnpo/residual_abs": 0.11951833963394165,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4857271493713894e-07,
|
|
"logits/chosen": -2.1099905967712402,
|
|
"logits/rejected": -2.1065971851348877,
|
|
"logps/chosen": -0.3332940936088562,
|
|
"logps/rejected": -0.33428648114204407,
|
|
"loss": 0.3267074227333069,
|
|
"loss/core": 0.32000210881233215,
|
|
"loss/preference_sft": 0.3332940936088562,
|
|
"loss/reference_anchor": 0.30193790793418884,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0046913623809814,
|
|
"rewards/margins": 1.1115453243255615,
|
|
"rewards/rejected": 0.8931459188461304,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13585230708122253,
|
|
"drift/rejected_abs": 0.12510886788368225,
|
|
"epoch": 0.12884753042233357,
|
|
"grad_norm": 105.0,
|
|
"ht_mnpo/logit": 0.010902149602770805,
|
|
"ht_mnpo/residual": 0.005902145989239216,
|
|
"ht_mnpo/residual_abs": 0.10268938541412354,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.481842300350339e-07,
|
|
"logits/chosen": -2.2717299461364746,
|
|
"logits/rejected": -2.1862072944641113,
|
|
"logps/chosen": -0.26431578397750854,
|
|
"logps/rejected": -0.2654600441455841,
|
|
"loss": 0.11229200661182404,
|
|
"loss/core": 0.10951416194438934,
|
|
"loss/preference_sft": 0.26431578397750854,
|
|
"loss/reference_anchor": 0.11246071010828018,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.3585233688354492,
|
|
"rewards/margins": 0.10902152210474014,
|
|
"rewards/rejected": 1.2495018243789673,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28132104873657227,
|
|
"drift/rejected_abs": 0.15722861886024475,
|
|
"epoch": 0.1336196611787163,
|
|
"grad_norm": 272.0,
|
|
"ht_mnpo/logit": 0.12429539859294891,
|
|
"ht_mnpo/residual": 0.1192954033613205,
|
|
"ht_mnpo/residual_abs": 0.16520704329013824,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4774942058187854e-07,
|
|
"logits/chosen": -2.127556324005127,
|
|
"logits/rejected": -2.184530735015869,
|
|
"logps/chosen": -0.29260799288749695,
|
|
"logps/rejected": -0.27409014105796814,
|
|
"loss": 0.3249160051345825,
|
|
"loss/core": 0.31713393330574036,
|
|
"loss/preference_sft": 0.29260799288749695,
|
|
"loss/reference_anchor": 0.3598441779613495,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8131933212280273,
|
|
"rewards/margins": 1.2429540157318115,
|
|
"rewards/rejected": 1.5702394247055054,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13580551743507385,
|
|
"drift/rejected_abs": 0.08986364305019379,
|
|
"epoch": 0.138391791935099,
|
|
"grad_norm": 0.50390625,
|
|
"ht_mnpo/logit": 0.046709753572940826,
|
|
"ht_mnpo/residual": 0.04170975089073181,
|
|
"ht_mnpo/residual_abs": 0.09605139493942261,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.472684500917257e-07,
|
|
"logits/chosen": -1.9042880535125732,
|
|
"logits/rejected": -1.8750311136245728,
|
|
"logps/chosen": -0.3126002550125122,
|
|
"logps/rejected": -0.3028576076030731,
|
|
"loss": 0.08231665939092636,
|
|
"loss/core": 0.08015841245651245,
|
|
"loss/preference_sft": 0.3126002550125122,
|
|
"loss/reference_anchor": 0.07665302604436874,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.3563940525054932,
|
|
"rewards/margins": 0.46709758043289185,
|
|
"rewards/rejected": 0.8892965316772461,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25080087780952454,
|
|
"drift/rejected_abs": 0.18363021314144135,
|
|
"epoch": 0.14316392269148176,
|
|
"grad_norm": 85.5,
|
|
"ht_mnpo/logit": 0.06734486669301987,
|
|
"ht_mnpo/residual": 0.06234486773610115,
|
|
"ht_mnpo/residual_abs": 0.19637265801429749,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.467414994379065e-07,
|
|
"logits/chosen": -2.0690462589263916,
|
|
"logits/rejected": -2.0414555072784424,
|
|
"logps/chosen": -0.2994147539138794,
|
|
"logps/rejected": -0.31211525201797485,
|
|
"loss": 0.49615511298179626,
|
|
"loss/core": 0.4883514940738678,
|
|
"loss/preference_sft": 0.2994147539138794,
|
|
"loss/reference_anchor": 0.36023765802383423,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.507319211959839,
|
|
"rewards/margins": 0.6734486818313599,
|
|
"rewards/rejected": 1.8338706493377686,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16852137446403503,
|
|
"drift/rejected_abs": 0.14129950106143951,
|
|
"epoch": 0.14793605344786448,
|
|
"grad_norm": 4.96875,
|
|
"ht_mnpo/logit": 0.028056615963578224,
|
|
"ht_mnpo/residual": 0.023056630045175552,
|
|
"ht_mnpo/residual_abs": 0.12251297384500504,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4616876678501114e-07,
|
|
"logits/chosen": -2.131013870239258,
|
|
"logits/rejected": -2.1533591747283936,
|
|
"logps/chosen": -0.28131020069122314,
|
|
"logps/rejected": -0.2708612084388733,
|
|
"loss": 0.22561495006084442,
|
|
"loss/core": 0.2211950123310089,
|
|
"loss/preference_sft": 0.28131020069122314,
|
|
"loss/reference_anchor": 0.1928669959306717,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6846176385879517,
|
|
"rewards/margins": 0.2805662751197815,
|
|
"rewards/rejected": 1.404051423072815,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.320243775844574,
|
|
"drift/rejected_abs": 0.16745874285697937,
|
|
"epoch": 0.1527081842042472,
|
|
"grad_norm": 222.0,
|
|
"ht_mnpo/logit": 0.15298235416412354,
|
|
"ht_mnpo/residual": 0.14798232913017273,
|
|
"ht_mnpo/residual_abs": 0.25980624556541443,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4555046751436735e-07,
|
|
"logits/chosen": -2.047654390335083,
|
|
"logits/rejected": -2.106396198272705,
|
|
"logps/chosen": -0.28943127393722534,
|
|
"logps/rejected": -0.2772199213504791,
|
|
"loss": 0.6820236444473267,
|
|
"loss/core": 0.6723781824111938,
|
|
"loss/preference_sft": 0.28943127393722534,
|
|
"loss/reference_anchor": 0.45332932472229004,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.20243763923645,
|
|
"rewards/margins": 1.5298234224319458,
|
|
"rewards/rejected": 1.672614336013794,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27144092321395874,
|
|
"drift/rejected_abs": 0.1204414814710617,
|
|
"epoch": 0.15748031496062992,
|
|
"grad_norm": 2.3125,
|
|
"ht_mnpo/logit": 0.15102508664131165,
|
|
"ht_mnpo/residual": 0.14602509140968323,
|
|
"ht_mnpo/residual_abs": 0.1496870219707489,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4488683414304425e-07,
|
|
"logits/chosen": -2.335982322692871,
|
|
"logits/rejected": -2.3015449047088623,
|
|
"logps/chosen": -0.29247352480888367,
|
|
"logps/rejected": -0.27914655208587646,
|
|
"loss": 0.26414981484413147,
|
|
"loss/core": 0.25607991218566895,
|
|
"loss/preference_sft": 0.29247352480888367,
|
|
"loss/reference_anchor": 0.37424594163894653,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.713765859603882,
|
|
"rewards/margins": 1.5102509260177612,
|
|
"rewards/rejected": 1.2035149335861206,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17270924150943756,
|
|
"drift/rejected_abs": 0.1425841748714447,
|
|
"epoch": 0.16225244571701264,
|
|
"grad_norm": 1.3046875,
|
|
"ht_mnpo/logit": 0.030281126499176025,
|
|
"ht_mnpo/residual": 0.025281131267547607,
|
|
"ht_mnpo/residual_abs": 0.09685656428337097,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4417811623641203e-07,
|
|
"logits/chosen": -2.147362470626831,
|
|
"logits/rejected": -2.008695363998413,
|
|
"logps/chosen": -0.2702954113483429,
|
|
"logps/rejected": -0.2617551386356354,
|
|
"loss": 0.09857013821601868,
|
|
"loss/core": 0.09527895599603653,
|
|
"loss/preference_sft": 0.2702954113483429,
|
|
"loss/reference_anchor": 0.13752929866313934,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.7254279851913452,
|
|
"rewards/margins": 0.30281129479408264,
|
|
"rewards/rejected": 1.4226164817810059,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13901950418949127,
|
|
"drift/rejected_abs": 0.06285737454891205,
|
|
"epoch": 0.16702457647339536,
|
|
"grad_norm": 27.75,
|
|
"ht_mnpo/logit": 0.07633363455533981,
|
|
"ht_mnpo/residual": 0.0713336318731308,
|
|
"ht_mnpo/residual_abs": 0.0760313868522644,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4342458031429113e-07,
|
|
"logits/chosen": -2.2728612422943115,
|
|
"logits/rejected": -2.179896831512451,
|
|
"logps/chosen": -0.28125789761543274,
|
|
"logps/rejected": -0.2593241333961487,
|
|
"loss": 0.06590215116739273,
|
|
"loss/core": 0.06366212666034698,
|
|
"loss/preference_sft": 0.28125789761543274,
|
|
"loss/reference_anchor": 0.08387625217437744,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.3895217180252075,
|
|
"rewards/margins": 0.7633363008499146,
|
|
"rewards/rejected": 0.6261853575706482,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1901485025882721,
|
|
"drift/rejected_abs": 0.10939119011163712,
|
|
"epoch": 0.1717967072297781,
|
|
"grad_norm": 3.984375,
|
|
"ht_mnpo/logit": 0.08394713699817657,
|
|
"ht_mnpo/residual": 0.07894712686538696,
|
|
"ht_mnpo/residual_abs": 0.08561943471431732,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4262650975072444e-07,
|
|
"logits/chosen": -2.1660208702087402,
|
|
"logits/rejected": -2.2384419441223145,
|
|
"logps/chosen": -0.29047879576683044,
|
|
"logps/rejected": -0.27046507596969604,
|
|
"loss": 0.04060132056474686,
|
|
"loss/core": 0.037876274436712265,
|
|
"loss/preference_sft": 0.29047879576683044,
|
|
"loss/reference_anchor": 0.10720425844192505,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.8998758792877197,
|
|
"rewards/margins": 0.8394712209701538,
|
|
"rewards/rejected": 1.0604045391082764,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2142634391784668,
|
|
"drift/rejected_abs": 0.09474581480026245,
|
|
"epoch": 0.17656883798616083,
|
|
"grad_norm": 2.0,
|
|
"ht_mnpo/logit": 0.12820716202259064,
|
|
"ht_mnpo/residual": 0.12320715188980103,
|
|
"ht_mnpo/residual_abs": 0.12510161101818085,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.417842046674122e-07,
|
|
"logits/chosen": -1.956552267074585,
|
|
"logits/rejected": -2.041656017303467,
|
|
"logps/chosen": -0.28232455253601074,
|
|
"logps/rejected": -0.28129279613494873,
|
|
"loss": 0.13155274093151093,
|
|
"loss/core": 0.12529942393302917,
|
|
"loss/preference_sft": 0.28232455253601074,
|
|
"loss/reference_anchor": 0.2844333052635193,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1409764289855957,
|
|
"rewards/margins": 1.2820714712142944,
|
|
"rewards/rejected": 0.8589048385620117,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1476023942232132,
|
|
"drift/rejected_abs": 0.09407086670398712,
|
|
"epoch": 0.18134096874254355,
|
|
"grad_norm": 2.3125,
|
|
"ht_mnpo/logit": 0.05354578420519829,
|
|
"ht_mnpo/residual": 0.048545777797698975,
|
|
"ht_mnpo/residual_abs": 0.07173924148082733,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4089798182084843e-07,
|
|
"logits/chosen": -2.1769192218780518,
|
|
"logits/rejected": -2.289309024810791,
|
|
"logps/chosen": -0.29876869916915894,
|
|
"logps/rejected": -0.29231029748916626,
|
|
"loss": 0.029190903529524803,
|
|
"loss/core": 0.027344604954123497,
|
|
"loss/preference_sft": 0.29876869916915894,
|
|
"loss/reference_anchor": 0.06243804842233658,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4751077890396118,
|
|
"rewards/margins": 0.5354577302932739,
|
|
"rewards/rejected": 0.9396499395370483,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29235774278640747,
|
|
"drift/rejected_abs": 0.13147327303886414,
|
|
"epoch": 0.18611309949892627,
|
|
"grad_norm": 14.4375,
|
|
"ht_mnpo/logit": 0.16157792508602142,
|
|
"ht_mnpo/residual": 0.1565779149532318,
|
|
"ht_mnpo/residual_abs": 0.18083927035331726,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3996817448320195e-07,
|
|
"logits/chosen": -2.084517240524292,
|
|
"logits/rejected": -2.0568737983703613,
|
|
"logps/chosen": -0.29337945580482483,
|
|
"logps/rejected": -0.29407113790512085,
|
|
"loss": 0.17335663735866547,
|
|
"loss/core": 0.16855856776237488,
|
|
"loss/preference_sft": 0.29337945580482483,
|
|
"loss/reference_anchor": 0.21056561172008514,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.920558214187622,
|
|
"rewards/margins": 1.6157792806625366,
|
|
"rewards/rejected": 1.304778814315796,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22568628191947937,
|
|
"drift/rejected_abs": 0.11775574833154678,
|
|
"epoch": 0.19088523025530899,
|
|
"grad_norm": 2.796875,
|
|
"ht_mnpo/logit": 0.10790988057851791,
|
|
"ht_mnpo/residual": 0.1029098853468895,
|
|
"ht_mnpo/residual_abs": 0.10873430967330933,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3899513231698607e-07,
|
|
"logits/chosen": -2.2028231620788574,
|
|
"logits/rejected": -2.1879143714904785,
|
|
"logps/chosen": -0.2985078692436218,
|
|
"logps/rejected": -0.30355304479599,
|
|
"loss": 0.24885663390159607,
|
|
"loss/core": 0.24319525063037872,
|
|
"loss/preference_sft": 0.2985078692436218,
|
|
"loss/reference_anchor": 0.25322026014328003,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.256082057952881,
|
|
"rewards/margins": 1.0790985822677612,
|
|
"rewards/rejected": 1.17698335647583,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1635308712720871,
|
|
"drift/rejected_abs": 0.11117591708898544,
|
|
"epoch": 0.19565736101169173,
|
|
"grad_norm": 12.125,
|
|
"ht_mnpo/logit": 0.05233384296298027,
|
|
"ht_mnpo/residual": 0.047333844006061554,
|
|
"ht_mnpo/residual_abs": 0.0785122960805893,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3797922124356506e-07,
|
|
"logits/chosen": -1.996665596961975,
|
|
"logits/rejected": -2.086698293685913,
|
|
"logps/chosen": -0.3164716362953186,
|
|
"logps/rejected": -0.3085149824619293,
|
|
"loss": 0.044618308544158936,
|
|
"loss/core": 0.040909815579652786,
|
|
"loss/preference_sft": 0.3164716362953186,
|
|
"loss/reference_anchor": 0.1537773758172989,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.631773591041565,
|
|
"rewards/margins": 0.5233384370803833,
|
|
"rewards/rejected": 1.1084349155426025,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3185950815677643,
|
|
"drift/rejected_abs": 0.16004785895347595,
|
|
"epoch": 0.20042949176807445,
|
|
"grad_norm": 12.875,
|
|
"ht_mnpo/logit": 0.15874508023262024,
|
|
"ht_mnpo/residual": 0.15374508500099182,
|
|
"ht_mnpo/residual_abs": 0.17046402394771576,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3692082330554585e-07,
|
|
"logits/chosen": -1.9498332738876343,
|
|
"logits/rejected": -1.9200170040130615,
|
|
"logps/chosen": -0.2881793677806854,
|
|
"logps/rejected": -0.2933315336704254,
|
|
"loss": 0.2835574746131897,
|
|
"loss/core": 0.27446287870407104,
|
|
"loss/preference_sft": 0.2881793677806854,
|
|
"loss/reference_anchor": 0.4259108006954193,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.185950756072998,
|
|
"rewards/margins": 1.5874507427215576,
|
|
"rewards/rejected": 1.59850013256073,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20830459892749786,
|
|
"drift/rejected_abs": 0.12604334950447083,
|
|
"epoch": 0.20520162252445717,
|
|
"grad_norm": 1.0078125,
|
|
"ht_mnpo/logit": 0.0821927934885025,
|
|
"ht_mnpo/residual": 0.07719278335571289,
|
|
"ht_mnpo/residual_abs": 0.12102246284484863,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3582033652310765e-07,
|
|
"logits/chosen": -2.1301636695861816,
|
|
"logits/rejected": -2.1746068000793457,
|
|
"logps/chosen": -0.2960442304611206,
|
|
"logps/rejected": -0.2779351472854614,
|
|
"loss": 0.1845921128988266,
|
|
"loss/core": 0.17939044535160065,
|
|
"loss/preference_sft": 0.2960442304611206,
|
|
"loss/reference_anchor": 0.23047873377799988,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.080594539642334,
|
|
"rewards/margins": 0.8219278454780579,
|
|
"rewards/rejected": 1.258666753768921,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19475789368152618,
|
|
"drift/rejected_abs": 0.13050904870033264,
|
|
"epoch": 0.2099737532808399,
|
|
"grad_norm": 0.76953125,
|
|
"ht_mnpo/logit": 0.06595344096422195,
|
|
"ht_mnpo/residual": 0.060953445732593536,
|
|
"ht_mnpo/residual_abs": 0.08129678666591644,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.346781747443227e-07,
|
|
"logits/chosen": -2.091243028640747,
|
|
"logits/rejected": -2.200920820236206,
|
|
"logps/chosen": -0.32776325941085815,
|
|
"logps/rejected": -0.3202073574066162,
|
|
"loss": 0.09159465879201889,
|
|
"loss/core": 0.08345116674900055,
|
|
"loss/preference_sft": 0.32776325941085815,
|
|
"loss/reference_anchor": 0.3743983805179596,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9458799362182617,
|
|
"rewards/margins": 0.6595343947410583,
|
|
"rewards/rejected": 1.286345362663269,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3241156041622162,
|
|
"drift/rejected_abs": 0.1684974730014801,
|
|
"epoch": 0.2147458840372226,
|
|
"grad_norm": 1.0234375,
|
|
"ht_mnpo/logit": 0.15598830580711365,
|
|
"ht_mnpo/residual": 0.15098831057548523,
|
|
"ht_mnpo/residual_abs": 0.18187597393989563,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3349476748952508e-07,
|
|
"logits/chosen": -2.1542420387268066,
|
|
"logits/rejected": -2.227787494659424,
|
|
"logps/chosen": -0.25393348932266235,
|
|
"logps/rejected": -0.23377032577991486,
|
|
"loss": 0.25889310240745544,
|
|
"loss/core": 0.24731948971748352,
|
|
"loss/preference_sft": 0.25393348932266235,
|
|
"loss/reference_anchor": 0.5532869100570679,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2385611534118652,
|
|
"rewards/margins": 1.5598829984664917,
|
|
"rewards/rejected": 1.6786781549453735,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11558188498020172,
|
|
"drift/rejected_abs": 0.06361546367406845,
|
|
"epoch": 0.21951801479360533,
|
|
"grad_norm": 4.625,
|
|
"ht_mnpo/logit": 0.052035581320524216,
|
|
"ht_mnpo/residual": 0.0470355823636055,
|
|
"ht_mnpo/residual_abs": 0.05875653028488159,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3227055978978545e-07,
|
|
"logits/chosen": -2.0761351585388184,
|
|
"logits/rejected": -2.168020009994507,
|
|
"logps/chosen": -0.319638729095459,
|
|
"logps/rejected": -0.2952752113342285,
|
|
"loss": 0.015921492129564285,
|
|
"loss/core": 0.014574600383639336,
|
|
"loss/preference_sft": 0.319638729095459,
|
|
"loss/reference_anchor": 0.03538065403699875,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.1521855592727661,
|
|
"rewards/margins": 0.5203558206558228,
|
|
"rewards/rejected": 0.6318296194076538,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.183929443359375,
|
|
"drift/rejected_abs": 0.12602423131465912,
|
|
"epoch": 0.22429014554998808,
|
|
"grad_norm": 4.28125,
|
|
"ht_mnpo/logit": 0.05796173959970474,
|
|
"ht_mnpo/residual": 0.05296173691749573,
|
|
"ht_mnpo/residual_abs": 0.1496523916721344,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3100601201955321e-07,
|
|
"logits/chosen": -2.0285463333129883,
|
|
"logits/rejected": -2.161771059036255,
|
|
"logps/chosen": -0.30416566133499146,
|
|
"logps/rejected": -0.29958218336105347,
|
|
"loss": 0.3189251124858856,
|
|
"loss/core": 0.3137461543083191,
|
|
"loss/preference_sft": 0.30416566133499146,
|
|
"loss/reference_anchor": 0.22853069007396698,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8382625579833984,
|
|
"rewards/margins": 0.5796173810958862,
|
|
"rewards/rejected": 1.2586452960968018,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15257421135902405,
|
|
"drift/rejected_abs": 0.09819750487804413,
|
|
"epoch": 0.2290622763063708,
|
|
"grad_norm": 2.9375,
|
|
"ht_mnpo/logit": 0.054583191871643066,
|
|
"ht_mnpo/residual": 0.04958319664001465,
|
|
"ht_mnpo/residual_abs": 0.06293803453445435,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.2970159972352864e-07,
|
|
"logits/chosen": -2.17802095413208,
|
|
"logits/rejected": -2.2032787799835205,
|
|
"logps/chosen": -0.29463058710098267,
|
|
"logps/rejected": -0.27752771973609924,
|
|
"loss": 0.01321292482316494,
|
|
"loss/core": 0.011416586115956306,
|
|
"loss/preference_sft": 0.29463058710098267,
|
|
"loss/reference_anchor": 0.060353875160217285,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5236282348632812,
|
|
"rewards/margins": 0.5458319783210754,
|
|
"rewards/rejected": 0.9777960777282715,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15810930728912354,
|
|
"drift/rejected_abs": 0.12099383026361465,
|
|
"epoch": 0.23383440706275352,
|
|
"grad_norm": 2.34375,
|
|
"ht_mnpo/logit": 0.03711375966668129,
|
|
"ht_mnpo/residual": 0.03211376070976257,
|
|
"ht_mnpo/residual_abs": 0.11317561566829681,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.2835781343782966e-07,
|
|
"logits/chosen": -2.1228532791137695,
|
|
"logits/rejected": -2.219285011291504,
|
|
"logps/chosen": -0.2916942536830902,
|
|
"logps/rejected": -0.2701827883720398,
|
|
"loss": 0.14501670002937317,
|
|
"loss/core": 0.14180776476860046,
|
|
"loss/preference_sft": 0.2916942536830902,
|
|
"loss/reference_anchor": 0.13127687573432922,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5794703960418701,
|
|
"rewards/margins": 0.3711375296115875,
|
|
"rewards/rejected": 1.2083330154418945,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3347860872745514,
|
|
"drift/rejected_abs": 0.12715332210063934,
|
|
"epoch": 0.23860653781913624,
|
|
"grad_norm": 47.75,
|
|
"ht_mnpo/logit": 0.20922037959098816,
|
|
"ht_mnpo/residual": 0.20422036945819855,
|
|
"ht_mnpo/residual_abs": 0.2054227888584137,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.2697515850552152e-07,
|
|
"logits/chosen": -1.8939529657363892,
|
|
"logits/rejected": -1.9978229999542236,
|
|
"logps/chosen": -0.30424895882606506,
|
|
"logps/rejected": -0.27813011407852173,
|
|
"loss": 0.33961981534957886,
|
|
"loss/core": 0.3286818861961365,
|
|
"loss/preference_sft": 0.30424895882606506,
|
|
"loss/reference_anchor": 0.5164695978164673,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.3477721214294434,
|
|
"rewards/margins": 2.0922036170959473,
|
|
"rewards/rejected": 1.2555686235427856,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2625555992126465,
|
|
"drift/rejected_abs": 0.16138434410095215,
|
|
"epoch": 0.24337866857551896,
|
|
"grad_norm": 4.625,
|
|
"ht_mnpo/logit": 0.10125492513179779,
|
|
"ht_mnpo/residual": 0.09625492990016937,
|
|
"ht_mnpo/residual_abs": 0.13495945930480957,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.2555415488657775e-07,
|
|
"logits/chosen": -1.8251349925994873,
|
|
"logits/rejected": -1.8897521495819092,
|
|
"logps/chosen": -0.299666166305542,
|
|
"logps/rejected": -0.29763779044151306,
|
|
"loss": 0.16235163807868958,
|
|
"loss/core": 0.15317954123020172,
|
|
"loss/preference_sft": 0.299666166305542,
|
|
"loss/reference_anchor": 0.42863768339157104,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.624504804611206,
|
|
"rewards/margins": 1.0125491619110107,
|
|
"rewards/rejected": 1.6119556427001953,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32063397765159607,
|
|
"drift/rejected_abs": 0.10358726978302002,
|
|
"epoch": 0.2481507993319017,
|
|
"grad_norm": 1368.0,
|
|
"ht_mnpo/logit": 0.21735945343971252,
|
|
"ht_mnpo/residual": 0.21235942840576172,
|
|
"ht_mnpo/residual_abs": 0.23501832783222198,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.240953369623447e-07,
|
|
"logits/chosen": -2.1691572666168213,
|
|
"logits/rejected": -2.276984691619873,
|
|
"logps/chosen": -0.3046864867210388,
|
|
"logps/rejected": -0.2934969365596771,
|
|
"loss": 1.100789189338684,
|
|
"loss/core": 1.0874181985855103,
|
|
"loss/preference_sft": 0.3046864867210388,
|
|
"loss/reference_anchor": 0.6380894780158997,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2051138877868652,
|
|
"rewards/margins": 2.1735942363739014,
|
|
"rewards/rejected": 1.0315196514129639,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.223398357629776,
|
|
"drift/rejected_abs": 0.20160217583179474,
|
|
"epoch": 0.2529229300882844,
|
|
"grad_norm": 32.0,
|
|
"ht_mnpo/logit": 0.03205033391714096,
|
|
"ht_mnpo/residual": 0.027050340548157692,
|
|
"ht_mnpo/residual_abs": 0.19905897974967957,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.225992533345824e-07,
|
|
"logits/chosen": -2.072179079055786,
|
|
"logits/rejected": -2.102553367614746,
|
|
"logps/chosen": -0.28394564986228943,
|
|
"logps/rejected": -0.2866360545158386,
|
|
"loss": 0.5659416317939758,
|
|
"loss/core": 0.5566951632499695,
|
|
"loss/preference_sft": 0.28394564986228943,
|
|
"loss/reference_anchor": 0.433927059173584,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2328720092773438,
|
|
"rewards/margins": 0.32050344347953796,
|
|
"rewards/rejected": 1.9123685359954834,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20549187064170837,
|
|
"drift/rejected_abs": 0.16975179314613342,
|
|
"epoch": 0.25769506084466715,
|
|
"grad_norm": 204.0,
|
|
"ht_mnpo/logit": 0.035853564739227295,
|
|
"ht_mnpo/residual": 0.03085356391966343,
|
|
"ht_mnpo/residual_abs": 0.19780030846595764,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.210664666191579e-07,
|
|
"logits/chosen": -2.068603038787842,
|
|
"logits/rejected": -2.148378372192383,
|
|
"logps/chosen": -0.2916108965873718,
|
|
"logps/rejected": -0.32321587204933167,
|
|
"loss": 0.5143053531646729,
|
|
"loss/core": 0.5063118934631348,
|
|
"loss/preference_sft": 0.2916108965873718,
|
|
"loss/reference_anchor": 0.37051326036453247,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0546793937683105,
|
|
"rewards/margins": 0.3585355281829834,
|
|
"rewards/rejected": 1.6961438655853271,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1933935433626175,
|
|
"drift/rejected_abs": 0.12514333426952362,
|
|
"epoch": 0.26246719160104987,
|
|
"grad_norm": 6.8125,
|
|
"ht_mnpo/logit": 0.06834165751934052,
|
|
"ht_mnpo/residual": 0.0633416622877121,
|
|
"ht_mnpo/residual_abs": 0.09675142914056778,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.1949755323446848e-07,
|
|
"logits/chosen": -2.070564031600952,
|
|
"logits/rejected": -2.177929162979126,
|
|
"logps/chosen": -0.2939522862434387,
|
|
"logps/rejected": -0.2786833941936493,
|
|
"loss": 0.05802987888455391,
|
|
"loss/core": 0.05398845672607422,
|
|
"loss/preference_sft": 0.2939522862434387,
|
|
"loss/reference_anchor": 0.17267616093158722,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9336230754852295,
|
|
"rewards/margins": 0.6834166646003723,
|
|
"rewards/rejected": 1.2502063512802124,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2726713716983795,
|
|
"drift/rejected_abs": 0.15814051032066345,
|
|
"epoch": 0.2672393223574326,
|
|
"grad_norm": 99.0,
|
|
"ht_mnpo/logit": 0.1145574226975441,
|
|
"ht_mnpo/residual": 0.10955740511417389,
|
|
"ht_mnpo/residual_abs": 0.22781553864479065,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.1789310318467426e-07,
|
|
"logits/chosen": -2.0886340141296387,
|
|
"logits/rejected": -2.1369686126708984,
|
|
"logps/chosen": -0.3071291744709015,
|
|
"logps/rejected": -0.29813405871391296,
|
|
"loss": 0.5179394483566284,
|
|
"loss/core": 0.5096560716629028,
|
|
"loss/preference_sft": 0.3071291744709015,
|
|
"loss/reference_anchor": 0.3834558427333832,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.725396156311035,
|
|
"rewards/margins": 1.1455739736557007,
|
|
"rewards/rejected": 1.5798218250274658,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17260298132896423,
|
|
"drift/rejected_abs": 0.16383907198905945,
|
|
"epoch": 0.2720114531138153,
|
|
"grad_norm": 1.578125,
|
|
"ht_mnpo/logit": 0.008781191892921925,
|
|
"ht_mnpo/residual": 0.0037811901420354843,
|
|
"ht_mnpo/residual_abs": 0.10902787744998932,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.1625371983782182e-07,
|
|
"logits/chosen": -2.233491897583008,
|
|
"logits/rejected": -2.1172218322753906,
|
|
"logps/chosen": -0.2896055579185486,
|
|
"logps/rejected": -0.33911457657814026,
|
|
"loss": 0.08416543900966644,
|
|
"loss/core": 0.08064934611320496,
|
|
"loss/preference_sft": 0.2896055579185486,
|
|
"loss/reference_anchor": 0.1468445360660553,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.7244901657104492,
|
|
"rewards/margins": 0.0878119170665741,
|
|
"rewards/rejected": 1.6366783380508423,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10990118980407715,
|
|
"drift/rejected_abs": 0.046395592391490936,
|
|
"epoch": 0.276783583870198,
|
|
"grad_norm": 49.75,
|
|
"ht_mnpo/logit": 0.06795798242092133,
|
|
"ht_mnpo/residual": 0.06295798718929291,
|
|
"ht_mnpo/residual_abs": 0.07829216122627258,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.14580019698942e-07,
|
|
"logits/chosen": -2.262953281402588,
|
|
"logits/rejected": -2.218596935272217,
|
|
"logps/chosen": -0.3364924490451813,
|
|
"logps/rejected": -0.33589696884155273,
|
|
"loss": 0.10862968862056732,
|
|
"loss/core": 0.10672261565923691,
|
|
"loss/preference_sft": 0.3364924490451813,
|
|
"loss/reference_anchor": 0.061704643070697784,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.0977586507797241,
|
|
"rewards/margins": 0.6795798540115356,
|
|
"rewards/rejected": 0.41817885637283325,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20892763137817383,
|
|
"drift/rejected_abs": 0.07657907903194427,
|
|
"epoch": 0.28155571462658074,
|
|
"grad_norm": 152.0,
|
|
"ht_mnpo/logit": 0.1324269324541092,
|
|
"ht_mnpo/residual": 0.12742695212364197,
|
|
"ht_mnpo/residual_abs": 0.1512664258480072,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.1287263217820773e-07,
|
|
"logits/chosen": -2.0495729446411133,
|
|
"logits/rejected": -2.147361993789673,
|
|
"logps/chosen": -0.3076549172401428,
|
|
"logps/rejected": -0.28194406628608704,
|
|
"loss": 0.2917906939983368,
|
|
"loss/core": 0.28768688440322876,
|
|
"loss/preference_sft": 0.3076549172401428,
|
|
"loss/reference_anchor": 0.17442592978477478,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0885672569274902,
|
|
"rewards/margins": 1.32426917552948,
|
|
"rewards/rejected": 0.7642977833747864,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2935689687728882,
|
|
"drift/rejected_abs": 0.19712750613689423,
|
|
"epoch": 0.2863278453829635,
|
|
"grad_norm": 5.90625,
|
|
"ht_mnpo/logit": 0.09644874185323715,
|
|
"ht_mnpo/residual": 0.09144874662160873,
|
|
"ht_mnpo/residual_abs": 0.16998650133609772,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.111321993542385e-07,
|
|
"logits/chosen": -2.0562853813171387,
|
|
"logits/rejected": -2.131844997406006,
|
|
"logps/chosen": -0.2610465884208679,
|
|
"logps/rejected": -0.25709906220436096,
|
|
"loss": 0.19292078912258148,
|
|
"loss/core": 0.17979469895362854,
|
|
"loss/preference_sft": 0.2610465884208679,
|
|
"loss/reference_anchor": 0.6302005052566528,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9351634979248047,
|
|
"rewards/margins": 0.9644874334335327,
|
|
"rewards/rejected": 1.9706761837005615,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22880244255065918,
|
|
"drift/rejected_abs": 0.14696647226810455,
|
|
"epoch": 0.29109997613934624,
|
|
"grad_norm": 6.78125,
|
|
"ht_mnpo/logit": 0.08201225101947784,
|
|
"ht_mnpo/residual": 0.07701224833726883,
|
|
"ht_mnpo/residual_abs": 0.08669502288103104,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0935937573264096e-07,
|
|
"logits/chosen": -1.9870030879974365,
|
|
"logits/rejected": -2.0139083862304688,
|
|
"logps/chosen": -0.30271345376968384,
|
|
"logps/rejected": -0.30806469917297363,
|
|
"loss": 0.04288022220134735,
|
|
"loss/core": 0.03761247545480728,
|
|
"loss/preference_sft": 0.30271345376968384,
|
|
"loss/reference_anchor": 0.23311614990234375,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2861876487731934,
|
|
"rewards/margins": 0.8201225399971008,
|
|
"rewards/rejected": 1.4660651683807373,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28174325823783875,
|
|
"drift/rejected_abs": 0.1894901990890503,
|
|
"epoch": 0.29587210689572896,
|
|
"grad_norm": 564.0,
|
|
"ht_mnpo/logit": 0.0924399122595787,
|
|
"ht_mnpo/residual": 0.08743990957736969,
|
|
"ht_mnpo/residual_abs": 0.23076701164245605,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0755482799987596e-07,
|
|
"logits/chosen": -1.9850540161132812,
|
|
"logits/rejected": -1.970975637435913,
|
|
"logps/chosen": -0.285608172416687,
|
|
"logps/rejected": -0.2967362403869629,
|
|
"loss": 0.7051917910575867,
|
|
"loss/core": 0.6910423040390015,
|
|
"loss/preference_sft": 0.285608172416687,
|
|
"loss/reference_anchor": 0.6789166927337646,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.8170597553253174,
|
|
"rewards/margins": 0.9243990778923035,
|
|
"rewards/rejected": 1.8926608562469482,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18879075348377228,
|
|
"drift/rejected_abs": 0.1482362002134323,
|
|
"epoch": 0.3006442376521117,
|
|
"grad_norm": 45.5,
|
|
"ht_mnpo/logit": 0.040583543479442596,
|
|
"ht_mnpo/residual": 0.03558354452252388,
|
|
"ht_mnpo/residual_abs": 0.1549256145954132,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0571923477254526e-07,
|
|
"logits/chosen": -2.215289354324341,
|
|
"logits/rejected": -2.188716411590576,
|
|
"logps/chosen": -0.313434898853302,
|
|
"logps/rejected": -0.29403433203697205,
|
|
"loss": 0.23106110095977783,
|
|
"loss/core": 0.22662413120269775,
|
|
"loss/preference_sft": 0.313434898853302,
|
|
"loss/reference_anchor": 0.19050461053848267,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.8869355916976929,
|
|
"rewards/margins": 0.40583547949790955,
|
|
"rewards/rejected": 1.4811002016067505,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2432427704334259,
|
|
"drift/rejected_abs": 0.13727743923664093,
|
|
"epoch": 0.3054163684084944,
|
|
"grad_norm": 21.625,
|
|
"ht_mnpo/logit": 0.10607069730758667,
|
|
"ht_mnpo/residual": 0.10107071697711945,
|
|
"ht_mnpo/residual_abs": 0.1325535625219345,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.038532863421914e-07,
|
|
"logits/chosen": -2.0528945922851562,
|
|
"logits/rejected": -1.9978628158569336,
|
|
"logps/chosen": -0.28710001707077026,
|
|
"logps/rejected": -0.29180365800857544,
|
|
"loss": 0.0885167121887207,
|
|
"loss/core": 0.08476362377405167,
|
|
"loss/preference_sft": 0.28710001707077026,
|
|
"loss/reference_anchor": 0.15894415974617004,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4323668479919434,
|
|
"rewards/margins": 1.0607070922851562,
|
|
"rewards/rejected": 1.3716596364974976,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17099711298942566,
|
|
"drift/rejected_abs": 0.07577967643737793,
|
|
"epoch": 0.3101884991648771,
|
|
"grad_norm": 4.28125,
|
|
"ht_mnpo/logit": 0.09501691162586212,
|
|
"ht_mnpo/residual": 0.0900169089436531,
|
|
"ht_mnpo/residual_abs": 0.09240968525409698,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0195768441570726e-07,
|
|
"logits/chosen": -1.9887382984161377,
|
|
"logits/rejected": -2.1444485187530518,
|
|
"logps/chosen": -0.2988322377204895,
|
|
"logps/rejected": -0.2900279462337494,
|
|
"loss": 0.08121752738952637,
|
|
"loss/core": 0.07862718403339386,
|
|
"loss/preference_sft": 0.2988322377204895,
|
|
"loss/reference_anchor": 0.09963365644216537,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7076162099838257,
|
|
"rewards/margins": 0.9501692056655884,
|
|
"rewards/rejected": 0.7574471831321716,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15594327449798584,
|
|
"drift/rejected_abs": 0.07677812874317169,
|
|
"epoch": 0.31496062992125984,
|
|
"grad_norm": 3.203125,
|
|
"ht_mnpo/logit": 0.07974691689014435,
|
|
"ht_mnpo/residual": 0.07474692165851593,
|
|
"ht_mnpo/residual_abs": 0.08458121865987778,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0003314185145307e-07,
|
|
"logits/chosen": -2.0846686363220215,
|
|
"logits/rejected": -2.14108943939209,
|
|
"logps/chosen": -0.31113189458847046,
|
|
"logps/rejected": -0.2992934584617615,
|
|
"loss": 0.07248958945274353,
|
|
"loss/core": 0.0704246237874031,
|
|
"loss/preference_sft": 0.31113189458847046,
|
|
"loss/reference_anchor": 0.07213523983955383,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5589559078216553,
|
|
"rewards/margins": 0.7974691390991211,
|
|
"rewards/rejected": 0.7614867091178894,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23571383953094482,
|
|
"drift/rejected_abs": 0.13885119557380676,
|
|
"epoch": 0.31973276067764256,
|
|
"grad_norm": 1.6640625,
|
|
"ht_mnpo/logit": 0.09817638248205185,
|
|
"ht_mnpo/residual": 0.09317637979984283,
|
|
"ht_mnpo/residual_abs": 0.1573963165283203,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.9808038239117913e-07,
|
|
"logits/chosen": -2.0466277599334717,
|
|
"logits/rejected": -1.9438968896865845,
|
|
"logps/chosen": -0.31259217858314514,
|
|
"logps/rejected": -0.3062000274658203,
|
|
"loss": 0.20101185142993927,
|
|
"loss/core": 0.1956087350845337,
|
|
"loss/preference_sft": 0.31259217858314514,
|
|
"loss/reference_anchor": 0.2388957291841507,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3562703132629395,
|
|
"rewards/margins": 0.9817637205123901,
|
|
"rewards/rejected": 1.3745065927505493,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19340157508850098,
|
|
"drift/rejected_abs": 0.18404799699783325,
|
|
"epoch": 0.3245048914340253,
|
|
"grad_norm": 1.203125,
|
|
"ht_mnpo/logit": 0.009365958161652088,
|
|
"ht_mnpo/residual": 0.004365948028862476,
|
|
"ht_mnpo/residual_abs": 0.15316197276115417,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.9610014038785646e-07,
|
|
"logits/chosen": -2.0774543285369873,
|
|
"logits/rejected": -2.0459485054016113,
|
|
"logps/chosen": -0.3146652579307556,
|
|
"logps/rejected": -0.3233223259449005,
|
|
"loss": 0.27519553899765015,
|
|
"loss/core": 0.2662979066371918,
|
|
"loss/preference_sft": 0.3146652579307556,
|
|
"loss/reference_anchor": 0.41341686248779297,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9339611530303955,
|
|
"rewards/margins": 0.09365954250097275,
|
|
"rewards/rejected": 1.840301513671875,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3130771517753601,
|
|
"drift/rejected_abs": 0.16737492382526398,
|
|
"epoch": 0.329277022190408,
|
|
"grad_norm": 129.0,
|
|
"ht_mnpo/logit": 0.14578819274902344,
|
|
"ht_mnpo/residual": 0.14078819751739502,
|
|
"ht_mnpo/residual_abs": 0.22993621230125427,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.9409316052951657e-07,
|
|
"logits/chosen": -1.9784408807754517,
|
|
"logits/rejected": -2.0493741035461426,
|
|
"logps/chosen": -0.3167511820793152,
|
|
"logps/rejected": -0.28369244933128357,
|
|
"loss": 0.50648033618927,
|
|
"loss/core": 0.49761754274368286,
|
|
"loss/preference_sft": 0.3167511820793152,
|
|
"loss/reference_anchor": 0.4114700257778168,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.130608320236206,
|
|
"rewards/margins": 1.4578818082809448,
|
|
"rewards/rejected": 1.6727266311645508,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3182355761528015,
|
|
"drift/rejected_abs": 0.13714584708213806,
|
|
"epoch": 0.3340491529467907,
|
|
"grad_norm": 129.0,
|
|
"ht_mnpo/logit": 0.1825825273990631,
|
|
"ht_mnpo/residual": 0.1775825321674347,
|
|
"ht_mnpo/residual_abs": 0.19932051002979279,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.920601975592047e-07,
|
|
"logits/chosen": -2.3250603675842285,
|
|
"logits/rejected": -2.3682284355163574,
|
|
"logps/chosen": -0.2681167423725128,
|
|
"logps/rejected": -0.27266380190849304,
|
|
"loss": 0.4750707745552063,
|
|
"loss/core": 0.4656985402107239,
|
|
"loss/preference_sft": 0.2681167423725128,
|
|
"loss/reference_anchor": 0.4418010115623474,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1823558807373047,
|
|
"rewards/margins": 1.8258254528045654,
|
|
"rewards/rejected": 1.3565304279327393,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2780304551124573,
|
|
"drift/rejected_abs": 0.21358676254749298,
|
|
"epoch": 0.3388212837031735,
|
|
"grad_norm": 756.0,
|
|
"ht_mnpo/logit": 0.06450818479061127,
|
|
"ht_mnpo/residual": 0.05950819328427315,
|
|
"ht_mnpo/residual_abs": 0.15815815329551697,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.900020159911519e-07,
|
|
"logits/chosen": -2.156029462814331,
|
|
"logits/rejected": -2.147246837615967,
|
|
"logps/chosen": -0.27569761872291565,
|
|
"logps/rejected": -0.27141404151916504,
|
|
"loss": 0.2398795187473297,
|
|
"loss/core": 0.22512921690940857,
|
|
"loss/preference_sft": 0.27569761872291565,
|
|
"loss/reference_anchor": 0.7099437713623047,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.7803046703338623,
|
|
"rewards/margins": 0.6450818181037903,
|
|
"rewards/rejected": 2.135222911834717,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3071724474430084,
|
|
"drift/rejected_abs": 0.14715437591075897,
|
|
"epoch": 0.3435934144595562,
|
|
"grad_norm": 716.0,
|
|
"ht_mnpo/logit": 0.16002245247364044,
|
|
"ht_mnpo/residual": 0.15502245724201202,
|
|
"ht_mnpo/residual_abs": 0.18939676880836487,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.879193898232725e-07,
|
|
"logits/chosen": -1.8401505947113037,
|
|
"logits/rejected": -1.8808279037475586,
|
|
"logps/chosen": -0.2977268099784851,
|
|
"logps/rejected": -0.30872827768325806,
|
|
"loss": 0.424351304769516,
|
|
"loss/core": 0.41371244192123413,
|
|
"loss/preference_sft": 0.2977268099784851,
|
|
"loss/reference_anchor": 0.5021710991859436,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0710623264312744,
|
|
"rewards/margins": 1.600224494934082,
|
|
"rewards/rejected": 1.4708378314971924,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13023678958415985,
|
|
"drift/rejected_abs": 0.08073797076940536,
|
|
"epoch": 0.34836554521593893,
|
|
"grad_norm": 27.25,
|
|
"ht_mnpo/logit": 0.04946478456258774,
|
|
"ht_mnpo/residual": 0.04446478560566902,
|
|
"ht_mnpo/residual_abs": 0.05108083412051201,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.8581310224609496e-07,
|
|
"logits/chosen": -2.1624913215637207,
|
|
"logits/rejected": -2.199889659881592,
|
|
"logps/chosen": -0.29786261916160583,
|
|
"logps/rejected": -0.2950283885002136,
|
|
"loss": 0.024249404668807983,
|
|
"loss/core": 0.022617992013692856,
|
|
"loss/preference_sft": 0.29786261916160583,
|
|
"loss/reference_anchor": 0.051784414798021317,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.3009334802627563,
|
|
"rewards/margins": 0.4946478307247162,
|
|
"rewards/rejected": 0.806285560131073,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22245018184185028,
|
|
"drift/rejected_abs": 0.1544378399848938,
|
|
"epoch": 0.35313767597232165,
|
|
"grad_norm": 8.0,
|
|
"ht_mnpo/logit": 0.06817840784788132,
|
|
"ht_mnpo/residual": 0.0631784051656723,
|
|
"ht_mnpo/residual_abs": 0.19337259232997894,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.8368394534823635e-07,
|
|
"logits/chosen": -2.1282296180725098,
|
|
"logits/rejected": -2.102940320968628,
|
|
"logps/chosen": -0.27162525057792664,
|
|
"logps/rejected": -0.25887078046798706,
|
|
"loss": 0.5373419523239136,
|
|
"loss/core": 0.5292584300041199,
|
|
"loss/preference_sft": 0.27162525057792664,
|
|
"loss/reference_anchor": 0.37701216340065,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.223367691040039,
|
|
"rewards/margins": 0.6817840933799744,
|
|
"rewards/rejected": 1.5415834188461304,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27047693729400635,
|
|
"drift/rejected_abs": 0.13434390723705292,
|
|
"epoch": 0.35790980672870437,
|
|
"grad_norm": 22.375,
|
|
"ht_mnpo/logit": 0.13634856045246124,
|
|
"ht_mnpo/residual": 0.13134853541851044,
|
|
"ht_mnpo/residual_abs": 0.16767247021198273,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.8153271981852968e-07,
|
|
"logits/chosen": -2.171160936355591,
|
|
"logits/rejected": -2.2762207984924316,
|
|
"logps/chosen": -0.25913965702056885,
|
|
"logps/rejected": -0.27620944380760193,
|
|
"loss": 0.2835163474082947,
|
|
"loss/core": 0.2761160731315613,
|
|
"loss/preference_sft": 0.25913965702056885,
|
|
"loss/reference_anchor": 0.3440994620323181,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.7047693729400635,
|
|
"rewards/margins": 1.36348557472229,
|
|
"rewards/rejected": 1.341283917427063,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36079439520835876,
|
|
"drift/rejected_abs": 0.11835791915655136,
|
|
"epoch": 0.3626819374850871,
|
|
"grad_norm": 7.90625,
|
|
"ht_mnpo/logit": 0.24685482680797577,
|
|
"ht_mnpo/residual": 0.24185481667518616,
|
|
"ht_mnpo/residual_abs": 0.25279420614242554,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7936023464491812e-07,
|
|
"logits/chosen": -1.9889020919799805,
|
|
"logits/rejected": -2.0030906200408936,
|
|
"logps/chosen": -0.3349910378456116,
|
|
"logps/rejected": -0.28726398944854736,
|
|
"loss": 0.6925952434539795,
|
|
"loss/core": 0.6821285486221313,
|
|
"loss/preference_sft": 0.3349910378456116,
|
|
"loss/reference_anchor": 0.4898358881473541,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.605802059173584,
|
|
"rewards/margins": 2.46854829788208,
|
|
"rewards/rejected": 1.1372535228729248,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.190070241689682,
|
|
"drift/rejected_abs": 0.11075327545404434,
|
|
"epoch": 0.3674540682414698,
|
|
"grad_norm": 15.875,
|
|
"ht_mnpo/logit": 0.07903822511434555,
|
|
"ht_mnpo/residual": 0.07403822243213654,
|
|
"ht_mnpo/residual_abs": 0.09887916594743729,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7716730681022723e-07,
|
|
"logits/chosen": -2.1092896461486816,
|
|
"logits/rejected": -2.122014284133911,
|
|
"logps/chosen": -0.2971669137477875,
|
|
"logps/rejected": -0.28943219780921936,
|
|
"loss": 0.060740817338228226,
|
|
"loss/core": 0.05780009552836418,
|
|
"loss/preference_sft": 0.2971669137477875,
|
|
"loss/reference_anchor": 0.11731908470392227,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.897173285484314,
|
|
"rewards/margins": 0.7903822660446167,
|
|
"rewards/rejected": 1.1067907810211182,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1997186690568924,
|
|
"drift/rejected_abs": 0.14593148231506348,
|
|
"epoch": 0.37222619899785253,
|
|
"grad_norm": 4.0,
|
|
"ht_mnpo/logit": 0.05384016036987305,
|
|
"ht_mnpo/residual": 0.04884016513824463,
|
|
"ht_mnpo/residual_abs": 0.12053972482681274,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7495476098493152e-07,
|
|
"logits/chosen": -2.2018589973449707,
|
|
"logits/rejected": -2.266772747039795,
|
|
"logps/chosen": -0.2876618206501007,
|
|
"logps/rejected": -0.28682073950767517,
|
|
"loss": 0.13210603594779968,
|
|
"loss/core": 0.1281919628381729,
|
|
"loss/preference_sft": 0.2876618206501007,
|
|
"loss/reference_anchor": 0.16693679988384247,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.9971866607666016,
|
|
"rewards/margins": 0.5384016036987305,
|
|
"rewards/rejected": 1.458785057067871,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4221646189689636,
|
|
"drift/rejected_abs": 0.19121377170085907,
|
|
"epoch": 0.37699832975423525,
|
|
"grad_norm": 200.0,
|
|
"ht_mnpo/logit": 0.23127570748329163,
|
|
"ht_mnpo/residual": 0.2262757271528244,
|
|
"ht_mnpo/residual_abs": 0.2536894679069519,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7272342921702937e-07,
|
|
"logits/chosen": -2.016627550125122,
|
|
"logits/rejected": -2.0958032608032227,
|
|
"logps/chosen": -0.2756597399711609,
|
|
"logps/rejected": -0.27037227153778076,
|
|
"loss": 0.5089841485023499,
|
|
"loss/core": 0.4903375506401062,
|
|
"loss/preference_sft": 0.2756597399711609,
|
|
"loss/reference_anchor": 0.9047657251358032,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.218891620635986,
|
|
"rewards/margins": 2.3127572536468506,
|
|
"rewards/rejected": 1.9061342477798462,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23051393032073975,
|
|
"drift/rejected_abs": 0.13780668377876282,
|
|
"epoch": 0.38177046051061797,
|
|
"grad_norm": 6.0625,
|
|
"ht_mnpo/logit": 0.09285277873277664,
|
|
"ht_mnpo/residual": 0.08785279095172882,
|
|
"ht_mnpo/residual_abs": 0.09289351105690002,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.7047415061914393e-07,
|
|
"logits/chosen": -2.1754462718963623,
|
|
"logits/rejected": -2.227482318878174,
|
|
"logps/chosen": -0.2999895215034485,
|
|
"logps/rejected": -0.2864961326122284,
|
|
"loss": 0.08253012597560883,
|
|
"loss/core": 0.0778684988617897,
|
|
"loss/preference_sft": 0.2999895215034485,
|
|
"loss/reference_anchor": 0.2030823677778244,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.304029941558838,
|
|
"rewards/margins": 0.9285279512405396,
|
|
"rewards/rejected": 1.3755018711090088,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2020423412322998,
|
|
"drift/rejected_abs": 0.07275157421827316,
|
|
"epoch": 0.3865425912670007,
|
|
"grad_norm": 9.4375,
|
|
"ht_mnpo/logit": 0.1292640119791031,
|
|
"ht_mnpo/residual": 0.12426400184631348,
|
|
"ht_mnpo/residual_abs": 0.12864765524864197,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.6820777105296707e-07,
|
|
"logits/chosen": -2.068509340286255,
|
|
"logits/rejected": -2.117534875869751,
|
|
"logps/chosen": -0.2823387682437897,
|
|
"logps/rejected": -0.27292540669441223,
|
|
"loss": 0.24370598793029785,
|
|
"loss/core": 0.23992788791656494,
|
|
"loss/preference_sft": 0.2823387682437897,
|
|
"loss/reference_anchor": 0.16066968441009521,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.018876075744629,
|
|
"rewards/margins": 1.292639970779419,
|
|
"rewards/rejected": 0.7262359857559204,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20162303745746613,
|
|
"drift/rejected_abs": 0.1299503743648529,
|
|
"epoch": 0.39131472202338347,
|
|
"grad_norm": 17.125,
|
|
"ht_mnpo/logit": 0.07248403877019882,
|
|
"ht_mnpo/residual": 0.0674840435385704,
|
|
"ht_mnpo/residual_abs": 0.1857225000858307,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.6592514281116553e-07,
|
|
"logits/chosen": -2.032942056655884,
|
|
"logits/rejected": -2.0350537300109863,
|
|
"logps/chosen": -0.29434436559677124,
|
|
"logps/rejected": -0.294363796710968,
|
|
"loss": 0.29699331521987915,
|
|
"loss/core": 0.2922920286655426,
|
|
"loss/preference_sft": 0.29434436559677124,
|
|
"loss/reference_anchor": 0.20562903583049774,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.014000654220581,
|
|
"rewards/margins": 0.7248404026031494,
|
|
"rewards/rejected": 1.2891602516174316,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3014175593852997,
|
|
"drift/rejected_abs": 0.1166742816567421,
|
|
"epoch": 0.3960868527797662,
|
|
"grad_norm": 3.265625,
|
|
"ht_mnpo/logit": 0.18487218022346497,
|
|
"ht_mnpo/residual": 0.17987218499183655,
|
|
"ht_mnpo/residual_abs": 0.1962762326002121,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.636271242968684e-07,
|
|
"logits/chosen": -1.9811691045761108,
|
|
"logits/rejected": -2.0673305988311768,
|
|
"logps/chosen": -0.30998358130455017,
|
|
"logps/rejected": -0.27552932500839233,
|
|
"loss": 0.6068626642227173,
|
|
"loss/core": 0.5962422490119934,
|
|
"loss/preference_sft": 0.30998358130455017,
|
|
"loss/reference_anchor": 0.5000226497650146,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0127615928649902,
|
|
"rewards/margins": 1.8487217426300049,
|
|
"rewards/rejected": 1.1640396118164062,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22677898406982422,
|
|
"drift/rejected_abs": 0.09225823730230331,
|
|
"epoch": 0.4008589835361489,
|
|
"grad_norm": 22.75,
|
|
"ht_mnpo/logit": 0.13460934162139893,
|
|
"ht_mnpo/residual": 0.1296093463897705,
|
|
"ht_mnpo/residual_abs": 0.13361980020999908,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.6131457970085684e-07,
|
|
"logits/chosen": -2.0054891109466553,
|
|
"logits/rejected": -2.080866813659668,
|
|
"logps/chosen": -0.2823740839958191,
|
|
"logps/rejected": -0.2677801251411438,
|
|
"loss": 0.1006707176566124,
|
|
"loss/core": 0.09769301116466522,
|
|
"loss/preference_sft": 0.2823740839958191,
|
|
"loss/reference_anchor": 0.12064739316701889,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.2673637866973877,
|
|
"rewards/margins": 1.3460934162139893,
|
|
"rewards/rejected": 0.9212702512741089,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23399285972118378,
|
|
"drift/rejected_abs": 0.07922275364398956,
|
|
"epoch": 0.4056311142925316,
|
|
"grad_norm": 580.0,
|
|
"ht_mnpo/logit": 0.15503880381584167,
|
|
"ht_mnpo/residual": 0.15003880858421326,
|
|
"ht_mnpo/residual_abs": 0.16690881550312042,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.5898837867657727e-07,
|
|
"logits/chosen": -2.0042977333068848,
|
|
"logits/rejected": -2.1140353679656982,
|
|
"logps/chosen": -0.294498085975647,
|
|
"logps/rejected": -0.28271484375,
|
|
"loss": 0.5954475402832031,
|
|
"loss/core": 0.5873497128486633,
|
|
"loss/preference_sft": 0.294498085975647,
|
|
"loss/reference_anchor": 0.3754442036151886,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.338529109954834,
|
|
"rewards/margins": 1.550387978553772,
|
|
"rewards/rejected": 0.7881409525871277,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2102353274822235,
|
|
"drift/rejected_abs": 0.11086465418338776,
|
|
"epoch": 0.41040324504891434,
|
|
"grad_norm": 139.0,
|
|
"ht_mnpo/logit": 0.09947403520345688,
|
|
"ht_mnpo/residual": 0.09447403252124786,
|
|
"ht_mnpo/residual_abs": 0.14160969853401184,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.5664939601310023e-07,
|
|
"logits/chosen": -2.4258694648742676,
|
|
"logits/rejected": -2.3425300121307373,
|
|
"logps/chosen": -0.28239089250564575,
|
|
"logps/rejected": -0.2863374352455139,
|
|
"loss": 0.2521866261959076,
|
|
"loss/core": 0.24730908870697021,
|
|
"loss/preference_sft": 0.28239089250564575,
|
|
"loss/reference_anchor": 0.21563799679279327,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.102165699005127,
|
|
"rewards/margins": 0.9947404861450195,
|
|
"rewards/rejected": 1.1074254512786865,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24716107547283173,
|
|
"drift/rejected_abs": 0.13077858090400696,
|
|
"epoch": 0.41517537580529706,
|
|
"grad_norm": 165.0,
|
|
"ht_mnpo/logit": 0.11873073875904083,
|
|
"ht_mnpo/residual": 0.1137307733297348,
|
|
"ht_mnpo/residual_abs": 0.20443549752235413,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.5429851130614807e-07,
|
|
"logits/chosen": -2.232671022415161,
|
|
"logits/rejected": -2.221527338027954,
|
|
"logps/chosen": -0.2805400788784027,
|
|
"logps/rejected": -0.29566511511802673,
|
|
"loss": 0.9471338391304016,
|
|
"loss/core": 0.936263918876648,
|
|
"loss/preference_sft": 0.2805400788784027,
|
|
"loss/reference_anchor": 0.5154294967651367,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.470388889312744,
|
|
"rewards/margins": 1.187307596206665,
|
|
"rewards/rejected": 1.2830810546875,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2321050614118576,
|
|
"drift/rejected_abs": 0.14237116277217865,
|
|
"epoch": 0.4199475065616798,
|
|
"grad_norm": 28.625,
|
|
"ht_mnpo/logit": 0.08972425758838654,
|
|
"ht_mnpo/residual": 0.08472425490617752,
|
|
"ht_mnpo/residual_abs": 0.1322084367275238,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.51936608627315e-07,
|
|
"logits/chosen": -2.334120750427246,
|
|
"logits/rejected": -2.3476264476776123,
|
|
"logps/chosen": -0.2910197377204895,
|
|
"logps/rejected": -0.2775648832321167,
|
|
"loss": 0.1562577337026596,
|
|
"loss/core": 0.15094859898090363,
|
|
"loss/preference_sft": 0.2910197377204895,
|
|
"loss/reference_anchor": 0.23635432124137878,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.320913076400757,
|
|
"rewards/margins": 0.8972426652908325,
|
|
"rewards/rejected": 1.4236705303192139,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12170737981796265,
|
|
"drift/rejected_abs": 0.0694720596075058,
|
|
"epoch": 0.4247196373180625,
|
|
"grad_norm": 6.65625,
|
|
"ht_mnpo/logit": 0.05207798629999161,
|
|
"ht_mnpo/residual": 0.04707798734307289,
|
|
"ht_mnpo/residual_abs": 0.05085582658648491,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.4956457619160375e-07,
|
|
"logits/chosen": -2.181494951248169,
|
|
"logits/rejected": -2.2427611351013184,
|
|
"logps/chosen": -0.29907384514808655,
|
|
"logps/rejected": -0.29663634300231934,
|
|
"loss": 0.015848254784941673,
|
|
"loss/core": 0.014302779920399189,
|
|
"loss/preference_sft": 0.29907384514808655,
|
|
"loss/reference_anchor": 0.04736633971333504,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2155005931854248,
|
|
"rewards/margins": 0.5207798480987549,
|
|
"rewards/rejected": 0.6947206258773804,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1860462725162506,
|
|
"drift/rejected_abs": 0.11890468746423721,
|
|
"epoch": 0.4294917680744452,
|
|
"grad_norm": 10.125,
|
|
"ht_mnpo/logit": 0.06721819937229156,
|
|
"ht_mnpo/residual": 0.06221820041537285,
|
|
"ht_mnpo/residual_abs": 0.06938367336988449,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.471833060234044e-07,
|
|
"logits/chosen": -2.1414787769317627,
|
|
"logits/rejected": -2.154848337173462,
|
|
"logps/chosen": -0.283276230096817,
|
|
"logps/rejected": -0.27300840616226196,
|
|
"loss": 0.02786489762365818,
|
|
"loss/core": 0.024693552404642105,
|
|
"loss/preference_sft": 0.283276230096817,
|
|
"loss/reference_anchor": 0.13023975491523743,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8602144718170166,
|
|
"rewards/margins": 0.6721819043159485,
|
|
"rewards/rejected": 1.1880323886871338,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3013097643852234,
|
|
"drift/rejected_abs": 0.10819373279809952,
|
|
"epoch": 0.43426389883082794,
|
|
"grad_norm": 0.671875,
|
|
"ht_mnpo/logit": 0.1931832730770111,
|
|
"ht_mnpo/residual": 0.1881832778453827,
|
|
"ht_mnpo/residual_abs": 0.1936052143573761,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.4479369362104037e-07,
|
|
"logits/chosen": -2.102328062057495,
|
|
"logits/rejected": -2.152161121368408,
|
|
"logps/chosen": -0.28967249393463135,
|
|
"logps/rejected": -0.2850145399570465,
|
|
"loss": 0.6813478469848633,
|
|
"loss/core": 0.6696580648422241,
|
|
"loss/preference_sft": 0.28967249393463135,
|
|
"loss/reference_anchor": 0.5555153489112854,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.011138439178467,
|
|
"rewards/margins": 1.9318326711654663,
|
|
"rewards/rejected": 1.0793057680130005,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28658145666122437,
|
|
"drift/rejected_abs": 0.08983840048313141,
|
|
"epoch": 0.43903602958721066,
|
|
"grad_norm": 2608.0,
|
|
"ht_mnpo/logit": 0.19802699983119965,
|
|
"ht_mnpo/residual": 0.19302698969841003,
|
|
"ht_mnpo/residual_abs": 0.22204919159412384,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.4239663762000817e-07,
|
|
"logits/chosen": -2.2929089069366455,
|
|
"logits/rejected": -2.203780174255371,
|
|
"logps/chosen": -0.29892778396606445,
|
|
"logps/rejected": -0.28588932752609253,
|
|
"loss": 0.748153567314148,
|
|
"loss/core": 0.738308846950531,
|
|
"loss/preference_sft": 0.29892778396606445,
|
|
"loss/reference_anchor": 0.46233946084976196,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.865814685821533,
|
|
"rewards/margins": 1.9802697896957397,
|
|
"rewards/rejected": 0.8855449557304382,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15654951333999634,
|
|
"drift/rejected_abs": 0.09482669830322266,
|
|
"epoch": 0.44380816034359344,
|
|
"grad_norm": 10.5625,
|
|
"ht_mnpo/logit": 0.061750371009111404,
|
|
"ht_mnpo/residual": 0.056750379502773285,
|
|
"ht_mnpo/residual_abs": 0.07501949369907379,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3999303945503747e-07,
|
|
"logits/chosen": -2.1819443702697754,
|
|
"logits/rejected": -2.1801505088806152,
|
|
"logps/chosen": -0.29756730794906616,
|
|
"logps/rejected": -0.27815189957618713,
|
|
"loss": 0.029046928510069847,
|
|
"loss/core": 0.02667873539030552,
|
|
"loss/preference_sft": 0.29756730794906616,
|
|
"loss/reference_anchor": 0.08865313977003098,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5641218423843384,
|
|
"rewards/margins": 0.6175037026405334,
|
|
"rewards/rejected": 0.946618378162384,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30324727296829224,
|
|
"drift/rejected_abs": 0.1545564830303192,
|
|
"epoch": 0.44858029109997616,
|
|
"grad_norm": 19.25,
|
|
"ht_mnpo/logit": 0.1486632078886032,
|
|
"ht_mnpo/residual": 0.1436631977558136,
|
|
"ht_mnpo/residual_abs": 0.232595294713974,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3758380302109808e-07,
|
|
"logits/chosen": -1.9403120279312134,
|
|
"logits/rejected": -2.031698703765869,
|
|
"logps/chosen": -0.29452261328697205,
|
|
"logps/rejected": -0.28670722246170044,
|
|
"loss": 0.5407153367996216,
|
|
"loss/core": 0.531051516532898,
|
|
"loss/preference_sft": 0.29452261328697205,
|
|
"loss/reference_anchor": 0.45374003052711487,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0309526920318604,
|
|
"rewards/margins": 1.4866321086883545,
|
|
"rewards/rejected": 1.5443203449249268,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2222728729248047,
|
|
"drift/rejected_abs": 0.09268493205308914,
|
|
"epoch": 0.4533524218563589,
|
|
"grad_norm": 46.0,
|
|
"ht_mnpo/logit": 0.12961037456989288,
|
|
"ht_mnpo/residual": 0.12461037933826447,
|
|
"ht_mnpo/residual_abs": 0.14704172313213348,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.351698343334823e-07,
|
|
"logits/chosen": -2.151616334915161,
|
|
"logits/rejected": -2.2165794372558594,
|
|
"logps/chosen": -0.30637693405151367,
|
|
"logps/rejected": -0.2831774652004242,
|
|
"loss": 0.46157604455947876,
|
|
"loss/core": 0.4549567699432373,
|
|
"loss/preference_sft": 0.30637693405151367,
|
|
"loss/reference_anchor": 0.30032485723495483,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.2221248149871826,
|
|
"rewards/margins": 1.2961037158966064,
|
|
"rewards/rejected": 0.9260212182998657,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20173406600952148,
|
|
"drift/rejected_abs": 0.13807883858680725,
|
|
"epoch": 0.4581245526127416,
|
|
"grad_norm": 2.203125,
|
|
"ht_mnpo/logit": 0.06972820311784744,
|
|
"ht_mnpo/residual": 0.06472820043563843,
|
|
"ht_mnpo/residual_abs": 0.19808796048164368,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3275204118708942e-07,
|
|
"logits/chosen": -2.0868983268737793,
|
|
"logits/rejected": -2.093916416168213,
|
|
"logps/chosen": -0.2903420329093933,
|
|
"logps/rejected": -0.27758854627609253,
|
|
"loss": 0.48223456740379333,
|
|
"loss/core": 0.47636446356773376,
|
|
"loss/preference_sft": 0.2903420329093933,
|
|
"loss/reference_anchor": 0.26446932554244995,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.016468048095703,
|
|
"rewards/margins": 0.697282075881958,
|
|
"rewards/rejected": 1.3191860914230347,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3890398144721985,
|
|
"drift/rejected_abs": 0.14222189784049988,
|
|
"epoch": 0.4628966833691243,
|
|
"grad_norm": 1176.0,
|
|
"ht_mnpo/logit": 0.24679091572761536,
|
|
"ht_mnpo/residual": 0.24179089069366455,
|
|
"ht_mnpo/residual_abs": 0.24669161438941956,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3033133281504132e-07,
|
|
"logits/chosen": -2.0273680686950684,
|
|
"logits/rejected": -1.986276626586914,
|
|
"logps/chosen": -0.3349688649177551,
|
|
"logps/rejected": -0.3107163608074188,
|
|
"loss": 1.0060511827468872,
|
|
"loss/core": 0.9833115339279175,
|
|
"loss/preference_sft": 0.3349688649177551,
|
|
"loss/reference_anchor": 1.1034791469573975,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.8889527320861816,
|
|
"rewards/margins": 2.4679088592529297,
|
|
"rewards/rejected": 1.4210439920425415,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15872669219970703,
|
|
"drift/rejected_abs": 0.0995459258556366,
|
|
"epoch": 0.46766881412550704,
|
|
"grad_norm": 1.796875,
|
|
"ht_mnpo/logit": 0.059221915900707245,
|
|
"ht_mnpo/residual": 0.05422192066907883,
|
|
"ht_mnpo/residual_abs": 0.10469312965869904,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.2790861954675702e-07,
|
|
"logits/chosen": -2.0943925380706787,
|
|
"logits/rejected": -2.1815621852874756,
|
|
"logps/chosen": -0.29587632417678833,
|
|
"logps/rejected": -0.29876887798309326,
|
|
"loss": 0.09795178472995758,
|
|
"loss/core": 0.09527580440044403,
|
|
"loss/preference_sft": 0.29587632417678833,
|
|
"loss/reference_anchor": 0.10421168804168701,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5852258205413818,
|
|
"rewards/margins": 0.5922192335128784,
|
|
"rewards/rejected": 0.9930065870285034,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15152491629123688,
|
|
"drift/rejected_abs": 0.06521561741828918,
|
|
"epoch": 0.47244094488188976,
|
|
"grad_norm": 7.65625,
|
|
"ht_mnpo/logit": 0.08643212169408798,
|
|
"ht_mnpo/residual": 0.08143211901187897,
|
|
"ht_mnpo/residual_abs": 0.08807367831468582,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.2548481246561504e-07,
|
|
"logits/chosen": -2.2049636840820312,
|
|
"logits/rejected": -2.185023784637451,
|
|
"logps/chosen": -0.2812512516975403,
|
|
"logps/rejected": -0.27317750453948975,
|
|
"loss": 0.03975542262196541,
|
|
"loss/core": 0.037954915314912796,
|
|
"loss/preference_sft": 0.2812512516975403,
|
|
"loss/reference_anchor": 0.06190044432878494,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5122514963150024,
|
|
"rewards/margins": 0.864321231842041,
|
|
"rewards/rejected": 0.6479302048683167,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1506694257259369,
|
|
"drift/rejected_abs": 0.05255242437124252,
|
|
"epoch": 0.4772130756382725,
|
|
"grad_norm": 43.75,
|
|
"ht_mnpo/logit": 0.09834472835063934,
|
|
"ht_mnpo/residual": 0.09334473311901093,
|
|
"ht_mnpo/residual_abs": 0.09647145867347717,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.230608230663321e-07,
|
|
"logits/chosen": -2.069584369659424,
|
|
"logits/rejected": -2.1218812465667725,
|
|
"logps/chosen": -0.33464083075523376,
|
|
"logps/rejected": -0.31745070219039917,
|
|
"loss": 0.12984250485897064,
|
|
"loss/core": 0.12702324986457825,
|
|
"loss/preference_sft": 0.33464083075523376,
|
|
"loss/reference_anchor": 0.10749910771846771,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.5064592361450195,
|
|
"rewards/margins": 0.983447253704071,
|
|
"rewards/rejected": 0.5230119824409485,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2962421774864197,
|
|
"drift/rejected_abs": 0.14752820134162903,
|
|
"epoch": 0.4819852063946552,
|
|
"grad_norm": 3.375,
|
|
"ht_mnpo/logit": 0.1489817202091217,
|
|
"ht_mnpo/residual": 0.1439817100763321,
|
|
"ht_mnpo/residual_abs": 0.15336847305297852,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.206375629121874e-07,
|
|
"logits/chosen": -2.281447649002075,
|
|
"logits/rejected": -2.3297507762908936,
|
|
"logps/chosen": -0.2693077027797699,
|
|
"logps/rejected": -0.24752524495124817,
|
|
"loss": 0.31294775009155273,
|
|
"loss/core": 0.3029409945011139,
|
|
"loss/preference_sft": 0.2693077027797699,
|
|
"loss/reference_anchor": 0.47340482473373413,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.9623029232025146,
|
|
"rewards/margins": 1.4898172616958618,
|
|
"rewards/rejected": 1.4724856615066528,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16579356789588928,
|
|
"drift/rejected_abs": 0.13272346556186676,
|
|
"epoch": 0.4867573371510379,
|
|
"grad_norm": 1.421875,
|
|
"ht_mnpo/logit": 0.033492714166641235,
|
|
"ht_mnpo/residual": 0.02849271520972252,
|
|
"ht_mnpo/residual_abs": 0.10743961483240128,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.1821594329222079e-07,
|
|
"logits/chosen": -2.1006534099578857,
|
|
"logits/rejected": -2.0847597122192383,
|
|
"logps/chosen": -0.3140560984611511,
|
|
"logps/rejected": -0.3067092299461365,
|
|
"loss": 0.10225538164377213,
|
|
"loss/core": 0.09745608270168304,
|
|
"loss/preference_sft": 0.3140560984611511,
|
|
"loss/reference_anchor": 0.2085585594177246,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6566555500030518,
|
|
"rewards/margins": 0.33492717146873474,
|
|
"rewards/rejected": 1.3217281103134155,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3283791244029999,
|
|
"drift/rejected_abs": 0.14059999585151672,
|
|
"epoch": 0.49152946790742064,
|
|
"grad_norm": 4.96875,
|
|
"ht_mnpo/logit": 0.188704252243042,
|
|
"ht_mnpo/residual": 0.18370424211025238,
|
|
"ht_mnpo/residual_abs": 0.2544330060482025,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.157968748785344e-07,
|
|
"logits/chosen": -2.0187830924987793,
|
|
"logits/rejected": -2.02095365524292,
|
|
"logps/chosen": -0.27096566557884216,
|
|
"logps/rejected": -0.26615461707115173,
|
|
"loss": 0.6095878481864929,
|
|
"loss/core": 0.600264310836792,
|
|
"loss/preference_sft": 0.27096566557884216,
|
|
"loss/reference_anchor": 0.4390795826911926,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2837913036346436,
|
|
"rewards/margins": 1.8870424032211304,
|
|
"rewards/rejected": 1.3967489004135132,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1408330649137497,
|
|
"drift/rejected_abs": 0.1474970579147339,
|
|
"epoch": 0.4963015986638034,
|
|
"grad_norm": 7.71875,
|
|
"ht_mnpo/logit": -0.0067166597582399845,
|
|
"ht_mnpo/residual": -0.011716673150658607,
|
|
"ht_mnpo/residual_abs": 0.13443736732006073,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.1338126738382597e-07,
|
|
"logits/chosen": -2.0800325870513916,
|
|
"logits/rejected": -2.005817413330078,
|
|
"logps/chosen": -0.3182372450828552,
|
|
"logps/rejected": -0.29156559705734253,
|
|
"loss": 0.3686665892601013,
|
|
"loss/core": 0.3629279136657715,
|
|
"loss/preference_sft": 0.3182372450828552,
|
|
"loss/reference_anchor": 0.2551078200340271,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.406805157661438,
|
|
"rewards/margins": -0.06716668605804443,
|
|
"rewards/rejected": 1.4739716053009033,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10101151466369629,
|
|
"drift/rejected_abs": 0.07907047867774963,
|
|
"epoch": 0.5010737294201861,
|
|
"grad_norm": 0.765625,
|
|
"ht_mnpo/logit": 0.021959036588668823,
|
|
"ht_mnpo/residual": 0.016959037631750107,
|
|
"ht_mnpo/residual_abs": 0.063358373939991,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.1097002921928316e-07,
|
|
"logits/chosen": -2.3136537075042725,
|
|
"logits/rejected": -2.301931142807007,
|
|
"logps/chosen": -0.31532853841781616,
|
|
"logps/rejected": -0.2992285192012787,
|
|
"loss": 0.032750409096479416,
|
|
"loss/core": 0.031238410621881485,
|
|
"loss/preference_sft": 0.31532853841781616,
|
|
"loss/reference_anchor": 0.04406752064824104,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.0089143514633179,
|
|
"rewards/margins": 0.21959033608436584,
|
|
"rewards/rejected": 0.789324164390564,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22263045608997345,
|
|
"drift/rejected_abs": 0.17575280368328094,
|
|
"epoch": 0.5058458601765689,
|
|
"grad_norm": 344.0,
|
|
"ht_mnpo/logit": 0.04713992029428482,
|
|
"ht_mnpo/residual": 0.042139921337366104,
|
|
"ht_mnpo/residual_abs": 0.13901717960834503,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.0856406715296717e-07,
|
|
"logits/chosen": -2.194549798965454,
|
|
"logits/rejected": -2.0708463191986084,
|
|
"logps/chosen": -0.2923962473869324,
|
|
"logps/rejected": -0.28995051980018616,
|
|
"loss": 0.18184489011764526,
|
|
"loss/core": 0.17629767954349518,
|
|
"loss/preference_sft": 0.2923962473869324,
|
|
"loss/reference_anchor": 0.2481221854686737,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2255985736846924,
|
|
"rewards/margins": 0.4713992178440094,
|
|
"rewards/rejected": 1.7541992664337158,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3698323965072632,
|
|
"drift/rejected_abs": 0.1586240977048874,
|
|
"epoch": 0.5106179909329516,
|
|
"grad_norm": 1008.0,
|
|
"ht_mnpo/logit": 0.21117202937602997,
|
|
"ht_mnpo/residual": 0.20617201924324036,
|
|
"ht_mnpo/residual_abs": 0.2581304907798767,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.061642859688146e-07,
|
|
"logits/chosen": -2.0579802989959717,
|
|
"logits/rejected": -2.1108455657958984,
|
|
"logps/chosen": -0.29436683654785156,
|
|
"logps/rejected": -0.2859746217727661,
|
|
"loss": 1.2907676696777344,
|
|
"loss/core": 1.2714307308197021,
|
|
"loss/preference_sft": 0.29436683654785156,
|
|
"loss/reference_anchor": 0.9374252557754517,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.6979613304138184,
|
|
"rewards/margins": 2.111720323562622,
|
|
"rewards/rejected": 1.5862407684326172,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16676244139671326,
|
|
"drift/rejected_abs": 0.11376915872097015,
|
|
"epoch": 0.5153901216893343,
|
|
"grad_norm": 1.6953125,
|
|
"ht_mnpo/logit": 0.05305640026926994,
|
|
"ht_mnpo/residual": 0.048056405037641525,
|
|
"ht_mnpo/residual_abs": 0.07481013983488083,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.0377158812638491e-07,
|
|
"logits/chosen": -2.155945062637329,
|
|
"logits/rejected": -2.191561460494995,
|
|
"logps/chosen": -0.2934812903404236,
|
|
"logps/rejected": -0.2616683840751648,
|
|
"loss": 0.029054682701826096,
|
|
"loss/core": 0.02675912342965603,
|
|
"loss/preference_sft": 0.2934812903404236,
|
|
"loss/reference_anchor": 0.08542972803115845,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6675370931625366,
|
|
"rewards/margins": 0.5305640697479248,
|
|
"rewards/rejected": 1.1369731426239014,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19116166234016418,
|
|
"drift/rejected_abs": 0.09059781581163406,
|
|
"epoch": 0.520162252445717,
|
|
"grad_norm": 0.67578125,
|
|
"ht_mnpo/logit": 0.10065528005361557,
|
|
"ht_mnpo/residual": 0.09565527737140656,
|
|
"ht_mnpo/residual_abs": 0.10974618047475815,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.0138687342148249e-07,
|
|
"logits/chosen": -2.0369791984558105,
|
|
"logits/rejected": -2.043282985687256,
|
|
"logps/chosen": -0.29164591431617737,
|
|
"logps/rejected": -0.2742694914340973,
|
|
"loss": 0.20747418701648712,
|
|
"loss/core": 0.20390582084655762,
|
|
"loss/preference_sft": 0.29164591431617737,
|
|
"loss/reference_anchor": 0.1492522656917572,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.9103397130966187,
|
|
"rewards/margins": 1.0065529346466064,
|
|
"rewards/rejected": 0.903786838054657,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.214359313249588,
|
|
"drift/rejected_abs": 0.11399934440851212,
|
|
"epoch": 0.5249343832020997,
|
|
"grad_norm": 1.8359375,
|
|
"ht_mnpo/logit": 0.10017386823892593,
|
|
"ht_mnpo/residual": 0.09517386555671692,
|
|
"ht_mnpo/residual_abs": 0.13996323943138123,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.90110386477801e-08,
|
|
"logits/chosen": -2.2847418785095215,
|
|
"logits/rejected": -2.3595545291900635,
|
|
"logps/chosen": -0.3020949959754944,
|
|
"logps/rejected": -0.29033786058425903,
|
|
"loss": 0.21369631588459015,
|
|
"loss/core": 0.2089644968509674,
|
|
"loss/preference_sft": 0.3020949959754944,
|
|
"loss/reference_anchor": 0.2063818722963333,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.1406989097595215,
|
|
"rewards/margins": 1.0017387866973877,
|
|
"rewards/rejected": 1.1389601230621338,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18578848242759705,
|
|
"drift/rejected_abs": 0.11730077117681503,
|
|
"epoch": 0.5297065139584825,
|
|
"grad_norm": 10.6875,
|
|
"ht_mnpo/logit": 0.06949036568403244,
|
|
"ht_mnpo/residual": 0.06449036300182343,
|
|
"ht_mnpo/residual_abs": 0.10903620719909668,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.664497725957164e-08,
|
|
"logits/chosen": -2.161120891571045,
|
|
"logits/rejected": -2.135941982269287,
|
|
"logps/chosen": -0.2531643509864807,
|
|
"logps/rejected": -0.2631688117980957,
|
|
"loss": 0.09490617364645004,
|
|
"loss/core": 0.09214997291564941,
|
|
"loss/preference_sft": 0.2531643509864807,
|
|
"loss/reference_anchor": 0.11249303817749023,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8578850030899048,
|
|
"rewards/margins": 0.6949037313461304,
|
|
"rewards/rejected": 1.1629812717437744,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34908127784729004,
|
|
"drift/rejected_abs": 0.12885652482509613,
|
|
"epoch": 0.5344786447148652,
|
|
"grad_norm": 24.5,
|
|
"ht_mnpo/logit": 0.22048592567443848,
|
|
"ht_mnpo/residual": 0.21548593044281006,
|
|
"ht_mnpo/residual_abs": 0.21782401204109192,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.428957903578045e-08,
|
|
"logits/chosen": -2.149848699569702,
|
|
"logits/rejected": -2.2165474891662598,
|
|
"logps/chosen": -0.2862304747104645,
|
|
"logps/rejected": -0.26955145597457886,
|
|
"loss": 0.7289299368858337,
|
|
"loss/core": 0.7147458791732788,
|
|
"loss/preference_sft": 0.2862304747104645,
|
|
"loss/reference_anchor": 0.6805819869041443,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.490812301635742,
|
|
"rewards/margins": 2.2048592567443848,
|
|
"rewards/rejected": 1.285953402519226,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3812643587589264,
|
|
"drift/rejected_abs": 0.156807541847229,
|
|
"epoch": 0.5392507754712479,
|
|
"grad_norm": 1024.0,
|
|
"ht_mnpo/logit": 0.2333361804485321,
|
|
"ht_mnpo/residual": 0.2283361852169037,
|
|
"ht_mnpo/residual_abs": 0.3491974174976349,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.194572974534976e-08,
|
|
"logits/chosen": -2.1060054302215576,
|
|
"logits/rejected": -2.117906332015991,
|
|
"logps/chosen": -0.2742339074611664,
|
|
"logps/rejected": -0.3001040816307068,
|
|
"loss": 1.6279220581054688,
|
|
"loss/core": 1.6076691150665283,
|
|
"loss/preference_sft": 0.2742339074611664,
|
|
"loss/reference_anchor": 0.9852186441421509,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.8111414909362793,
|
|
"rewards/margins": 2.3333616256713867,
|
|
"rewards/rejected": 1.4777801036834717,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3591713607311249,
|
|
"drift/rejected_abs": 0.15439173579216003,
|
|
"epoch": 0.5440229062276306,
|
|
"grad_norm": 0.796875,
|
|
"ht_mnpo/logit": 0.2359723299741745,
|
|
"ht_mnpo/residual": 0.23097236454486847,
|
|
"ht_mnpo/residual_abs": 0.241423562169075,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.96143108141412e-08,
|
|
"logits/chosen": -2.0038273334503174,
|
|
"logits/rejected": -1.9924920797348022,
|
|
"logps/chosen": -0.28397005796432495,
|
|
"logps/rejected": -0.2956791818141937,
|
|
"loss": 0.6347066164016724,
|
|
"loss/core": 0.6228178143501282,
|
|
"loss/preference_sft": 0.28397005796432495,
|
|
"loss/reference_anchor": 0.566045880317688,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.590536117553711,
|
|
"rewards/margins": 2.3597235679626465,
|
|
"rewards/rejected": 1.230812668800354,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22817322611808777,
|
|
"drift/rejected_abs": 0.11515506356954575,
|
|
"epoch": 0.5487950369840133,
|
|
"grad_norm": 2.5,
|
|
"ht_mnpo/logit": 0.11322301626205444,
|
|
"ht_mnpo/residual": 0.10822302103042603,
|
|
"ht_mnpo/residual_abs": 0.11970474570989609,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.72961989934668e-08,
|
|
"logits/chosen": -2.163687229156494,
|
|
"logits/rejected": -2.2072675228118896,
|
|
"logps/chosen": -0.30935314297676086,
|
|
"logps/rejected": -0.2823563814163208,
|
|
"loss": 0.10977880656719208,
|
|
"loss/core": 0.10459931939840317,
|
|
"loss/preference_sft": 0.30935314297676086,
|
|
"loss/reference_anchor": 0.22803907096385956,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.281670331954956,
|
|
"rewards/margins": 1.132230281829834,
|
|
"rewards/rejected": 1.1494401693344116,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19220417737960815,
|
|
"drift/rejected_abs": 0.0809175968170166,
|
|
"epoch": 0.553567167740396,
|
|
"grad_norm": 3.015625,
|
|
"ht_mnpo/logit": 0.11272404342889786,
|
|
"ht_mnpo/residual": 0.10772404819726944,
|
|
"ht_mnpo/residual_abs": 0.13539370894432068,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.499226603037854e-08,
|
|
"logits/chosen": -2.2288291454315186,
|
|
"logits/rejected": -2.188666582107544,
|
|
"logps/chosen": -0.2963632345199585,
|
|
"logps/rejected": -0.2874986529350281,
|
|
"loss": 0.29833313822746277,
|
|
"loss/core": 0.29411810636520386,
|
|
"loss/preference_sft": 0.2963632345199585,
|
|
"loss/reference_anchor": 0.18111544847488403,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.922041654586792,
|
|
"rewards/margins": 1.1272404193878174,
|
|
"rewards/rejected": 0.7948012948036194,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21872587502002716,
|
|
"drift/rejected_abs": 0.1608632504940033,
|
|
"epoch": 0.5583392984967788,
|
|
"grad_norm": 0.443359375,
|
|
"ht_mnpo/logit": 0.057856976985931396,
|
|
"ht_mnpo/residual": 0.05285697430372238,
|
|
"ht_mnpo/residual_abs": 0.09607581794261932,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.270337833983987e-08,
|
|
"logits/chosen": -1.9375085830688477,
|
|
"logits/rejected": -1.851293921470642,
|
|
"logps/chosen": -0.27032381296157837,
|
|
"logps/rejected": -0.25147318840026855,
|
|
"loss": 0.044498372822999954,
|
|
"loss/core": 0.04125569015741348,
|
|
"loss/preference_sft": 0.27032381296157837,
|
|
"loss/reference_anchor": 0.13510197401046753,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.185845375061035,
|
|
"rewards/margins": 0.5785697102546692,
|
|
"rewards/rejected": 1.6072756052017212,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21919722855091095,
|
|
"drift/rejected_abs": 0.16523535549640656,
|
|
"epoch": 0.5631114292531615,
|
|
"grad_norm": 8.4375,
|
|
"ht_mnpo/logit": 0.05521801859140396,
|
|
"ht_mnpo/residual": 0.05021803453564644,
|
|
"ht_mnpo/residual_abs": 0.17762021720409393,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.04303966789025e-08,
|
|
"logits/chosen": -2.237201452255249,
|
|
"logits/rejected": -2.175689697265625,
|
|
"logps/chosen": -0.2361256182193756,
|
|
"logps/rejected": -0.2443951815366745,
|
|
"loss": 0.3992035984992981,
|
|
"loss/core": 0.39220893383026123,
|
|
"loss/preference_sft": 0.2361256182193756,
|
|
"loss/reference_anchor": 0.3261222839355469,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1915810108184814,
|
|
"rewards/margins": 0.5521801114082336,
|
|
"rewards/rejected": 1.6394007205963135,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14474406838417053,
|
|
"drift/rejected_abs": 0.07125679403543472,
|
|
"epoch": 0.5678835600095442,
|
|
"grad_norm": 0.2109375,
|
|
"ht_mnpo/logit": 0.07342871278524399,
|
|
"ht_mnpo/residual": 0.06842872500419617,
|
|
"ht_mnpo/residual_abs": 0.08838275074958801,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 7.817417582301098e-08,
|
|
"logits/chosen": -2.1800007820129395,
|
|
"logits/rejected": -2.2645437717437744,
|
|
"logps/chosen": -0.2769630551338196,
|
|
"logps/rejected": -0.27484917640686035,
|
|
"loss": 0.047025274485349655,
|
|
"loss/core": 0.04545805603265762,
|
|
"loss/preference_sft": 0.2769630551338196,
|
|
"loss/reference_anchor": 0.05066481977701187,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.4456117153167725,
|
|
"rewards/margins": 0.7342872619628906,
|
|
"rewards/rejected": 0.7113243937492371,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29197508096694946,
|
|
"drift/rejected_abs": 0.22344262897968292,
|
|
"epoch": 0.572655690765927,
|
|
"grad_norm": 9.1875,
|
|
"ht_mnpo/logit": 0.06846772134304047,
|
|
"ht_mnpo/residual": 0.06346772611141205,
|
|
"ht_mnpo/residual_abs": 0.2963734269142151,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 7.59355642445566e-08,
|
|
"logits/chosen": -2.0200035572052,
|
|
"logits/rejected": -2.113144636154175,
|
|
"logps/chosen": -0.27107173204421997,
|
|
"logps/rejected": -0.26764634251594543,
|
|
"loss": 1.2578901052474976,
|
|
"loss/core": 1.242497444152832,
|
|
"loss/preference_sft": 0.27107173204421997,
|
|
"loss/reference_anchor": 0.7425199747085571,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9191036224365234,
|
|
"rewards/margins": 0.6846773624420166,
|
|
"rewards/rejected": 2.2344260215759277,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31894126534461975,
|
|
"drift/rejected_abs": 0.131717711687088,
|
|
"epoch": 0.5774278215223098,
|
|
"grad_norm": 113.5,
|
|
"ht_mnpo/logit": 0.1888471394777298,
|
|
"ht_mnpo/residual": 0.18384712934494019,
|
|
"ht_mnpo/residual_abs": 0.2077023684978485,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 7.37154037938015e-08,
|
|
"logits/chosen": -2.216761350631714,
|
|
"logits/rejected": -2.2498691082000732,
|
|
"logps/chosen": -0.31387758255004883,
|
|
"logps/rejected": -0.31475016474723816,
|
|
"loss": 0.5773728489875793,
|
|
"loss/core": 0.5683153867721558,
|
|
"loss/preference_sft": 0.31387758255004883,
|
|
"loss/reference_anchor": 0.42148226499557495,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1892428398132324,
|
|
"rewards/margins": 1.8884713649749756,
|
|
"rewards/rejected": 1.3007718324661255,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19281364977359772,
|
|
"drift/rejected_abs": 0.10097751766443253,
|
|
"epoch": 0.5821999522786925,
|
|
"grad_norm": 2.46875,
|
|
"ht_mnpo/logit": 0.09355486929416656,
|
|
"ht_mnpo/residual": 0.08855487406253815,
|
|
"ht_mnpo/residual_abs": 0.10233534872531891,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 7.151452938229325e-08,
|
|
"logits/chosen": -2.125278949737549,
|
|
"logits/rejected": -2.109140634536743,
|
|
"logps/chosen": -0.27237892150878906,
|
|
"logps/rejected": -0.28159099817276,
|
|
"loss": 0.11355199664831161,
|
|
"loss/core": 0.11056599766016006,
|
|
"loss/preference_sft": 0.27237892150878906,
|
|
"loss/reference_anchor": 0.12206175178289413,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9259014129638672,
|
|
"rewards/margins": 0.9355487823486328,
|
|
"rewards/rejected": 0.9903526306152344,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31643611192703247,
|
|
"drift/rejected_abs": 0.10367069393396378,
|
|
"epoch": 0.5869720830350752,
|
|
"grad_norm": 59.25,
|
|
"ht_mnpo/logit": 0.2132229506969452,
|
|
"ht_mnpo/residual": 0.20822295546531677,
|
|
"ht_mnpo/residual_abs": 0.2242036610841751,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.933376866888883e-08,
|
|
"logits/chosen": -2.1933255195617676,
|
|
"logits/rejected": -2.2344157695770264,
|
|
"logps/chosen": -0.3152548670768738,
|
|
"logps/rejected": -0.2855820655822754,
|
|
"loss": 0.7186287641525269,
|
|
"loss/core": 0.7079665660858154,
|
|
"loss/preference_sft": 0.3152548670768738,
|
|
"loss/reference_anchor": 0.5015829801559448,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.1643614768981934,
|
|
"rewards/margins": 2.1322293281555176,
|
|
"rewards/rejected": 1.0321319103240967,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18896803259849548,
|
|
"drift/rejected_abs": 0.14922986924648285,
|
|
"epoch": 0.5917442137914579,
|
|
"grad_norm": 35.25,
|
|
"ht_mnpo/logit": 0.039714448153972626,
|
|
"ht_mnpo/residual": 0.03471444919705391,
|
|
"ht_mnpo/residual_abs": 0.15567417442798615,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.717394174850605e-08,
|
|
"logits/chosen": -2.108137607574463,
|
|
"logits/rejected": -2.1779696941375732,
|
|
"logps/chosen": -0.2668481469154358,
|
|
"logps/rejected": -0.2571088671684265,
|
|
"loss": 0.14907851815223694,
|
|
"loss/core": 0.14530247449874878,
|
|
"loss/preference_sft": 0.2668481469154358,
|
|
"loss/reference_anchor": 0.1621183156967163,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.889099359512329,
|
|
"rewards/margins": 0.3971443772315979,
|
|
"rewards/rejected": 1.4919549226760864,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4413048326969147,
|
|
"drift/rejected_abs": 0.23714692890644073,
|
|
"epoch": 0.5965163445478406,
|
|
"grad_norm": 5.59375,
|
|
"ht_mnpo/logit": 0.20502714812755585,
|
|
"ht_mnpo/residual": 0.20002715289592743,
|
|
"ht_mnpo/residual_abs": 0.2597729563713074,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.503586084371926e-08,
|
|
"logits/chosen": -1.7817914485931396,
|
|
"logits/rejected": -1.8193590641021729,
|
|
"logps/chosen": -0.24516813457012177,
|
|
"logps/rejected": -0.23617276549339294,
|
|
"loss": 0.46254611015319824,
|
|
"loss/core": 0.4459630846977234,
|
|
"loss/preference_sft": 0.24516813457012177,
|
|
"loss/reference_anchor": 0.804632842540741,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.412737846374512,
|
|
"rewards/margins": 2.05027174949646,
|
|
"rewards/rejected": 2.3624656200408936,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2613522708415985,
|
|
"drift/rejected_abs": 0.12580320239067078,
|
|
"epoch": 0.6012884753042234,
|
|
"grad_norm": 1.984375,
|
|
"ht_mnpo/logit": 0.136060431599617,
|
|
"ht_mnpo/residual": 0.1310604363679886,
|
|
"ht_mnpo/residual_abs": 0.133602574467659,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.29203299993155e-08,
|
|
"logits/chosen": -2.093968629837036,
|
|
"logits/rejected": -2.1899936199188232,
|
|
"logps/chosen": -0.26807597279548645,
|
|
"logps/rejected": -0.26248809695243835,
|
|
"loss": 0.22901996970176697,
|
|
"loss/core": 0.22321954369544983,
|
|
"loss/preference_sft": 0.26807597279548645,
|
|
"loss/reference_anchor": 0.263213574886322,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.6133341789245605,
|
|
"rewards/margins": 1.3606044054031372,
|
|
"rewards/rejected": 1.2527296543121338,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14114494621753693,
|
|
"drift/rejected_abs": 0.09878765046596527,
|
|
"epoch": 0.6060606060606061,
|
|
"grad_norm": 16.625,
|
|
"ht_mnpo/logit": 0.04238896444439888,
|
|
"ht_mnpo/residual": 0.03738895803689957,
|
|
"ht_mnpo/residual_abs": 0.12258099019527435,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.082814477992656e-08,
|
|
"logits/chosen": -2.223665952682495,
|
|
"logits/rejected": -2.122124195098877,
|
|
"logps/chosen": -0.30093568563461304,
|
|
"logps/rejected": -0.27837997674942017,
|
|
"loss": 0.12603580951690674,
|
|
"loss/core": 0.12354050576686859,
|
|
"loss/preference_sft": 0.30093568563461304,
|
|
"loss/reference_anchor": 0.09467045962810516,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.410095453262329,
|
|
"rewards/margins": 0.4238895773887634,
|
|
"rewards/rejected": 0.9862058758735657,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2346729040145874,
|
|
"drift/rejected_abs": 0.20684631168842316,
|
|
"epoch": 0.6108327368169888,
|
|
"grad_norm": 57.0,
|
|
"ht_mnpo/logit": 0.027944982051849365,
|
|
"ht_mnpo/residual": 0.022944996133446693,
|
|
"ht_mnpo/residual_abs": 0.24229581654071808,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.87600919708494e-08,
|
|
"logits/chosen": -2.0714011192321777,
|
|
"logits/rejected": -2.0947563648223877,
|
|
"logps/chosen": -0.28593140840530396,
|
|
"logps/rejected": -0.2925289273262024,
|
|
"loss": 0.5171207785606384,
|
|
"loss/core": 0.5091261863708496,
|
|
"loss/preference_sft": 0.28593140840530396,
|
|
"loss/reference_anchor": 0.3711358904838562,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.345926284790039,
|
|
"rewards/margins": 0.279449999332428,
|
|
"rewards/rejected": 2.066476583480835,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32909509539604187,
|
|
"drift/rejected_abs": 0.15848024189472198,
|
|
"epoch": 0.6156048675733715,
|
|
"grad_norm": 2.78125,
|
|
"ht_mnpo/logit": 0.17075712978839874,
|
|
"ht_mnpo/residual": 0.16575713455677032,
|
|
"ht_mnpo/residual_abs": 0.18923448026180267,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.671694928216829e-08,
|
|
"logits/chosen": -2.1546435356140137,
|
|
"logits/rejected": -2.1636524200439453,
|
|
"logps/chosen": -0.2983279228210449,
|
|
"logps/rejected": -0.297065794467926,
|
|
"loss": 0.4782744348049164,
|
|
"loss/core": 0.463600218296051,
|
|
"loss/preference_sft": 0.2983279228210449,
|
|
"loss/reference_anchor": 0.7038805484771729,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2895309925079346,
|
|
"rewards/margins": 1.707571268081665,
|
|
"rewards/rejected": 1.5819594860076904,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22253522276878357,
|
|
"drift/rejected_abs": 0.11168678849935532,
|
|
"epoch": 0.6203769983297542,
|
|
"grad_norm": 35.0,
|
|
"ht_mnpo/logit": 0.1109624132514,
|
|
"ht_mnpo/residual": 0.10596241056919098,
|
|
"ht_mnpo/residual_abs": 0.11568085104227066,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.469948505629e-08,
|
|
"logits/chosen": -2.0919768810272217,
|
|
"logits/rejected": -2.1772122383117676,
|
|
"logps/chosen": -0.2706933617591858,
|
|
"logps/rejected": -0.2707112431526184,
|
|
"loss": 0.08376672863960266,
|
|
"loss/core": 0.07922674715518951,
|
|
"loss/preference_sft": 0.2706933617591858,
|
|
"loss/reference_anchor": 0.19992953538894653,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2252449989318848,
|
|
"rewards/margins": 1.1096241474151611,
|
|
"rewards/rejected": 1.1156206130981445,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39330363273620605,
|
|
"drift/rejected_abs": 0.14271840453147888,
|
|
"epoch": 0.625149129086137,
|
|
"grad_norm": 828.0,
|
|
"ht_mnpo/logit": 0.2761969566345215,
|
|
"ht_mnpo/residual": 0.2711969316005707,
|
|
"ht_mnpo/residual_abs": 0.31647664308547974,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.270845797900168e-08,
|
|
"logits/chosen": -2.03916335105896,
|
|
"logits/rejected": -2.0866518020629883,
|
|
"logps/chosen": -0.3057062327861786,
|
|
"logps/rejected": -0.313825398683548,
|
|
"loss": 1.5293443202972412,
|
|
"loss/core": 1.511008858680725,
|
|
"loss/preference_sft": 0.3057062327861786,
|
|
"loss/reference_anchor": 0.8862015008926392,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.932830810546875,
|
|
"rewards/margins": 2.761969804763794,
|
|
"rewards/rejected": 1.1708608865737915,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2743390202522278,
|
|
"drift/rejected_abs": 0.09815811365842819,
|
|
"epoch": 0.6299212598425197,
|
|
"grad_norm": 16.0,
|
|
"ht_mnpo/logit": 0.17759980261325836,
|
|
"ht_mnpo/residual": 0.17259980738162994,
|
|
"ht_mnpo/residual_abs": 0.17895349860191345,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.0744616794160104e-08,
|
|
"logits/chosen": -1.9069111347198486,
|
|
"logits/rejected": -1.9322376251220703,
|
|
"logps/chosen": -0.28189000487327576,
|
|
"logps/rejected": -0.2763403058052063,
|
|
"loss": 0.4070788025856018,
|
|
"loss/core": 0.4006664752960205,
|
|
"loss/preference_sft": 0.28189000487327576,
|
|
"loss/reference_anchor": 0.29242876172065735,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7418372631073,
|
|
"rewards/margins": 1.7759981155395508,
|
|
"rewards/rejected": 0.9658395648002625,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19634750485420227,
|
|
"drift/rejected_abs": 0.0983700379729271,
|
|
"epoch": 0.6346933905989024,
|
|
"grad_norm": 15.6875,
|
|
"ht_mnpo/logit": 0.09792444109916687,
|
|
"ht_mnpo/residual": 0.09292443841695786,
|
|
"ht_mnpo/residual_abs": 0.12199316918849945,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.880870002211963e-08,
|
|
"logits/chosen": -2.0750694274902344,
|
|
"logits/rejected": -2.129474401473999,
|
|
"logps/chosen": -0.28144150972366333,
|
|
"logps/rejected": -0.27089667320251465,
|
|
"loss": 0.09351025521755219,
|
|
"loss/core": 0.09088621288537979,
|
|
"loss/preference_sft": 0.28144150972366333,
|
|
"loss/reference_anchor": 0.1030580997467041,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9629449844360352,
|
|
"rewards/margins": 0.9792444109916687,
|
|
"rewards/rejected": 0.9837003946304321,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1535734385251999,
|
|
"drift/rejected_abs": 0.15276674926280975,
|
|
"epoch": 0.6394655213552851,
|
|
"grad_norm": 89.5,
|
|
"ht_mnpo/logit": 0.004352384712547064,
|
|
"ht_mnpo/residual": -0.000647619366645813,
|
|
"ht_mnpo/residual_abs": 0.15819282829761505,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.6901435682004996e-08,
|
|
"logits/chosen": -2.4462497234344482,
|
|
"logits/rejected": -2.402447462081909,
|
|
"logps/chosen": -0.28123825788497925,
|
|
"logps/rejected": -0.28739991784095764,
|
|
"loss": 0.2847086787223816,
|
|
"loss/core": 0.2802528142929077,
|
|
"loss/preference_sft": 0.28123825788497925,
|
|
"loss/reference_anchor": 0.19467081129550934,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.534631371498108,
|
|
"rewards/margins": 0.04352378845214844,
|
|
"rewards/rejected": 1.491107702255249,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25063109397888184,
|
|
"drift/rejected_abs": 0.18034696578979492,
|
|
"epoch": 0.6442376521116678,
|
|
"grad_norm": 85.0,
|
|
"ht_mnpo/logit": 0.07082972675561905,
|
|
"ht_mnpo/residual": 0.06582970917224884,
|
|
"ht_mnpo/residual_abs": 0.1985233575105667,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.502354101793314e-08,
|
|
"logits/chosen": -2.028636932373047,
|
|
"logits/rejected": -2.079133987426758,
|
|
"logps/chosen": -0.2808381915092468,
|
|
"logps/rejected": -0.2724851667881012,
|
|
"loss": 0.3324066400527954,
|
|
"loss/core": 0.3255481719970703,
|
|
"loss/preference_sft": 0.2808381915092468,
|
|
"loss/reference_anchor": 0.31483718752861023,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5056729316711426,
|
|
"rewards/margins": 0.7082973122596741,
|
|
"rewards/rejected": 1.7973754405975342,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2721858322620392,
|
|
"drift/rejected_abs": 0.16804562509059906,
|
|
"epoch": 0.6490097828680506,
|
|
"grad_norm": 12.4375,
|
|
"ht_mnpo/logit": 0.1044061928987503,
|
|
"ht_mnpo/residual": 0.09940619766712189,
|
|
"ht_mnpo/residual_abs": 0.10363922268152237,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.3175722229287034e-08,
|
|
"logits/chosen": -1.9131011962890625,
|
|
"logits/rejected": -1.9577534198760986,
|
|
"logps/chosen": -0.30100974440574646,
|
|
"logps/rejected": -0.2880757451057434,
|
|
"loss": 0.09681503474712372,
|
|
"loss/core": 0.08140072226524353,
|
|
"loss/preference_sft": 0.30100974440574646,
|
|
"loss/reference_anchor": 0.7406147718429565,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7206804752349854,
|
|
"rewards/margins": 1.0440620183944702,
|
|
"rewards/rejected": 1.6766185760498047,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24128063023090363,
|
|
"drift/rejected_abs": 0.1263791024684906,
|
|
"epoch": 0.6537819136244333,
|
|
"grad_norm": 16.125,
|
|
"ht_mnpo/logit": 0.11510924249887466,
|
|
"ht_mnpo/residual": 0.11010923236608505,
|
|
"ht_mnpo/residual_abs": 0.15428678691387177,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.135867420514276e-08,
|
|
"logits/chosen": -2.1793155670166016,
|
|
"logits/rejected": -2.2106401920318604,
|
|
"logps/chosen": -0.27883365750312805,
|
|
"logps/rejected": -0.3018958270549774,
|
|
"loss": 0.25008782744407654,
|
|
"loss/core": 0.24473276734352112,
|
|
"loss/preference_sft": 0.27883365750312805,
|
|
"loss/reference_anchor": 0.23987166583538055,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4123270511627197,
|
|
"rewards/margins": 1.151092290878296,
|
|
"rewards/rejected": 1.2612347602844238,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2428806573152542,
|
|
"drift/rejected_abs": 0.12732911109924316,
|
|
"epoch": 0.658554044380816,
|
|
"grad_norm": 192.0,
|
|
"ht_mnpo/logit": 0.11564892530441284,
|
|
"ht_mnpo/residual": 0.11064890772104263,
|
|
"ht_mnpo/residual_abs": 0.1703292578458786,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.957308026295035e-08,
|
|
"logits/chosen": -2.0042989253997803,
|
|
"logits/rejected": -1.9776769876480103,
|
|
"logps/chosen": -0.306735634803772,
|
|
"logps/rejected": -0.28427666425704956,
|
|
"loss": 0.25161948800086975,
|
|
"loss/core": 0.2459324598312378,
|
|
"loss/preference_sft": 0.306735634803772,
|
|
"loss/reference_anchor": 0.2536788880825043,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4288065433502197,
|
|
"rewards/margins": 1.1564891338348389,
|
|
"rewards/rejected": 1.2723174095153809,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2985590100288391,
|
|
"drift/rejected_abs": 0.10956205427646637,
|
|
"epoch": 0.6633261751371987,
|
|
"grad_norm": 872.0,
|
|
"ht_mnpo/logit": 0.18910077214241028,
|
|
"ht_mnpo/residual": 0.18410077691078186,
|
|
"ht_mnpo/residual_abs": 0.19521042704582214,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.7819611891566084e-08,
|
|
"logits/chosen": -2.094963312149048,
|
|
"logits/rejected": -2.1175599098205566,
|
|
"logps/chosen": -0.292602002620697,
|
|
"logps/rejected": -0.2823565602302551,
|
|
"loss": 0.9227179288864136,
|
|
"loss/core": 0.9114183187484741,
|
|
"loss/preference_sft": 0.292602002620697,
|
|
"loss/reference_anchor": 0.5357309579849243,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9835476875305176,
|
|
"rewards/margins": 1.891007661819458,
|
|
"rewards/rejected": 1.0925395488739014,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4045167863368988,
|
|
"drift/rejected_abs": 0.1837201565504074,
|
|
"epoch": 0.6680983058935814,
|
|
"grad_norm": 716.0,
|
|
"ht_mnpo/logit": 0.2208547294139862,
|
|
"ht_mnpo/residual": 0.2158547192811966,
|
|
"ht_mnpo/residual_abs": 0.29663529992103577,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.609892849873286e-08,
|
|
"logits/chosen": -2.0619771480560303,
|
|
"logits/rejected": -2.0216448307037354,
|
|
"logps/chosen": -0.3050151467323303,
|
|
"logps/rejected": -0.31216365098953247,
|
|
"loss": 0.9451198577880859,
|
|
"loss/core": 0.9246786236763,
|
|
"loss/preference_sft": 0.3050151467323303,
|
|
"loss/reference_anchor": 0.991558849811554,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.043200969696045,
|
|
"rewards/margins": 2.2085471153259277,
|
|
"rewards/rejected": 1.8346538543701172,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27926188707351685,
|
|
"drift/rejected_abs": 0.20713751018047333,
|
|
"epoch": 0.6728704366499642,
|
|
"grad_norm": 1.6875,
|
|
"ht_mnpo/logit": 0.07208569347858429,
|
|
"ht_mnpo/residual": 0.06708566844463348,
|
|
"ht_mnpo/residual_abs": 0.15458956360816956,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.4411677163103894e-08,
|
|
"logits/chosen": -2.2489805221557617,
|
|
"logits/rejected": -2.1912360191345215,
|
|
"logps/chosen": -0.2617301940917969,
|
|
"logps/rejected": -0.27545157074928284,
|
|
"loss": 0.31091055274009705,
|
|
"loss/core": 0.296908438205719,
|
|
"loss/preference_sft": 0.2617301940917969,
|
|
"loss/reference_anchor": 0.6739317178726196,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.791400194168091,
|
|
"rewards/margins": 0.7208568453788757,
|
|
"rewards/rejected": 2.0705432891845703,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30785104632377625,
|
|
"drift/rejected_abs": 0.19405075907707214,
|
|
"epoch": 0.677642567406347,
|
|
"grad_norm": 356.0,
|
|
"ht_mnpo/logit": 0.11398478597402573,
|
|
"ht_mnpo/residual": 0.10898479074239731,
|
|
"ht_mnpo/residual_abs": 0.26380276679992676,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.2758492390902945e-08,
|
|
"logits/chosen": -2.044776439666748,
|
|
"logits/rejected": -2.13199782371521,
|
|
"logps/chosen": -0.3172752261161804,
|
|
"logps/rejected": -0.2907639443874359,
|
|
"loss": 0.7584068775177002,
|
|
"loss/core": 0.7458577752113342,
|
|
"loss/preference_sft": 0.3172752261161804,
|
|
"loss/reference_anchor": 0.5957252979278564,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.077186346054077,
|
|
"rewards/margins": 1.139847755432129,
|
|
"rewards/rejected": 1.9373382329940796,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2990885376930237,
|
|
"drift/rejected_abs": 0.13457565009593964,
|
|
"epoch": 0.6824146981627297,
|
|
"grad_norm": 12.9375,
|
|
"ht_mnpo/logit": 0.16464099287986755,
|
|
"ht_mnpo/residual": 0.15964099764823914,
|
|
"ht_mnpo/residual_abs": 0.16764886677265167,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.11399958773126e-08,
|
|
"logits/chosen": -1.7991693019866943,
|
|
"logits/rejected": -1.8733018636703491,
|
|
"logps/chosen": -0.278984010219574,
|
|
"logps/rejected": -0.30155643820762634,
|
|
"loss": 0.2886965274810791,
|
|
"loss/core": 0.28276368975639343,
|
|
"loss/preference_sft": 0.278984010219574,
|
|
"loss/reference_anchor": 0.26874133944511414,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9902405738830566,
|
|
"rewards/margins": 1.6464097499847412,
|
|
"rewards/rejected": 1.3438305854797363,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3325175642967224,
|
|
"drift/rejected_abs": 0.11430977284908295,
|
|
"epoch": 0.6871868289191124,
|
|
"grad_norm": 2.671875,
|
|
"ht_mnpo/logit": 0.21893766522407532,
|
|
"ht_mnpo/residual": 0.2139376401901245,
|
|
"ht_mnpo/residual_abs": 0.21863511204719543,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.9556796272679972e-08,
|
|
"logits/chosen": -2.025402784347534,
|
|
"logits/rejected": -2.015165090560913,
|
|
"logps/chosen": -0.2916553318500519,
|
|
"logps/rejected": -0.2767946124076843,
|
|
"loss": 0.617830753326416,
|
|
"loss/core": 0.6057730317115784,
|
|
"loss/preference_sft": 0.2916553318500519,
|
|
"loss/reference_anchor": 0.5737227201461792,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.32487416267395,
|
|
"rewards/margins": 2.1893765926361084,
|
|
"rewards/rejected": 1.135497808456421,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2585774064064026,
|
|
"drift/rejected_abs": 0.17033758759498596,
|
|
"epoch": 0.6919589596754951,
|
|
"grad_norm": 1.078125,
|
|
"ht_mnpo/logit": 0.08847717195749283,
|
|
"ht_mnpo/residual": 0.08347717672586441,
|
|
"ht_mnpo/residual_abs": 0.19887696206569672,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.8009488953628215e-08,
|
|
"logits/chosen": -2.201829195022583,
|
|
"logits/rejected": -2.16338849067688,
|
|
"logps/chosen": -0.2900460362434387,
|
|
"logps/rejected": -0.29323217272758484,
|
|
"loss": 0.304708331823349,
|
|
"loss/core": 0.2987585663795471,
|
|
"loss/preference_sft": 0.2900460362434387,
|
|
"loss/reference_anchor": 0.2684846520423889,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.585101366043091,
|
|
"rewards/margins": 0.8847716450691223,
|
|
"rewards/rejected": 1.7003295421600342,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21931128203868866,
|
|
"drift/rejected_abs": 0.11406012624502182,
|
|
"epoch": 0.6967310904318779,
|
|
"grad_norm": 338.0,
|
|
"ht_mnpo/logit": 0.10529018938541412,
|
|
"ht_mnpo/residual": 0.1002901941537857,
|
|
"ht_mnpo/residual_abs": 0.11368098109960556,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.649865579915976e-08,
|
|
"logits/chosen": -2.1011104583740234,
|
|
"logits/rejected": -2.1871607303619385,
|
|
"logps/chosen": -0.30004754662513733,
|
|
"logps/rejected": -0.28622013330459595,
|
|
"loss": 0.19001969695091248,
|
|
"loss/core": 0.18584798276424408,
|
|
"loss/preference_sft": 0.30004754662513733,
|
|
"loss/reference_anchor": 0.17857913672924042,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.191316604614258,
|
|
"rewards/margins": 1.0529019832611084,
|
|
"rewards/rejected": 1.138414978981018,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12048070132732391,
|
|
"drift/rejected_abs": 0.19749411940574646,
|
|
"epoch": 0.7015032211882606,
|
|
"grad_norm": 151.0,
|
|
"ht_mnpo/logit": -0.07722408324480057,
|
|
"ht_mnpo/residual": -0.08222408592700958,
|
|
"ht_mnpo/residual_abs": 0.16358783841133118,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.5024864971835507e-08,
|
|
"logits/chosen": -2.2358853816986084,
|
|
"logits/rejected": -2.1690421104431152,
|
|
"logps/chosen": -0.3068314492702484,
|
|
"logps/rejected": -0.31617873907089233,
|
|
"loss": 0.2910768389701843,
|
|
"loss/core": 0.2859953045845032,
|
|
"loss/preference_sft": 0.3068314492702484,
|
|
"loss/reference_anchor": 0.2233956754207611,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.2027002573013306,
|
|
"rewards/margins": -0.7722408771514893,
|
|
"rewards/rejected": 1.9749412536621094,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.287181556224823,
|
|
"drift/rejected_abs": 0.21348950266838074,
|
|
"epoch": 0.7062753519446433,
|
|
"grad_norm": 9.625,
|
|
"ht_mnpo/logit": 0.07476375997066498,
|
|
"ht_mnpo/residual": 0.06976374983787537,
|
|
"ht_mnpo/residual_abs": 0.22156234085559845,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.3588670704111997e-08,
|
|
"logits/chosen": -2.060324192047119,
|
|
"logits/rejected": -1.9929609298706055,
|
|
"logps/chosen": -0.2671794891357422,
|
|
"logps/rejected": -0.28327035903930664,
|
|
"loss": 0.3700963854789734,
|
|
"loss/core": 0.36213165521621704,
|
|
"loss/preference_sft": 0.2671794891357422,
|
|
"loss/reference_anchor": 0.3715202808380127,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.8711066246032715,
|
|
"rewards/margins": 0.7476376295089722,
|
|
"rewards/rejected": 2.123469114303589,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2255537211894989,
|
|
"drift/rejected_abs": 0.08047862350940704,
|
|
"epoch": 0.711047482701026,
|
|
"grad_norm": 23.125,
|
|
"ht_mnpo/logit": 0.15348081290721893,
|
|
"ht_mnpo/residual": 0.14848081767559052,
|
|
"ht_mnpo/residual_abs": 0.15392430126667023,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.219061308991721e-08,
|
|
"logits/chosen": -2.09189772605896,
|
|
"logits/rejected": -2.121368885040283,
|
|
"logps/chosen": -0.27154573798179626,
|
|
"logps/rejected": -0.27765482664108276,
|
|
"loss": 0.21156100928783417,
|
|
"loss/core": 0.20837783813476562,
|
|
"loss/preference_sft": 0.27154573798179626,
|
|
"loss/reference_anchor": 0.13200418651103973,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.252340316772461,
|
|
"rewards/margins": 1.5348081588745117,
|
|
"rewards/rejected": 0.717532217502594,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24951839447021484,
|
|
"drift/rejected_abs": 0.0656016543507576,
|
|
"epoch": 0.7158196134574087,
|
|
"grad_norm": 14.25,
|
|
"ht_mnpo/logit": 0.1838001310825348,
|
|
"ht_mnpo/residual": 0.17880013585090637,
|
|
"ht_mnpo/residual_abs": 0.1926441490650177,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.0831217881543557e-08,
|
|
"logits/chosen": -2.2540266513824463,
|
|
"logits/rejected": -2.2791855335235596,
|
|
"logps/chosen": -0.31038394570350647,
|
|
"logps/rejected": -0.2928916811943054,
|
|
"loss": 0.9867032170295715,
|
|
"loss/core": 0.9744570851325989,
|
|
"loss/preference_sft": 0.31038394570350647,
|
|
"loss/reference_anchor": 0.5812680721282959,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4931938648223877,
|
|
"rewards/margins": 1.8380012512207031,
|
|
"rewards/rejected": 0.6551925539970398,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2845051884651184,
|
|
"drift/rejected_abs": 0.11132697761058807,
|
|
"epoch": 0.7205917442137915,
|
|
"grad_norm": 53.0,
|
|
"ht_mnpo/logit": 0.17366446554660797,
|
|
"ht_mnpo/residual": 0.16866444051265717,
|
|
"ht_mnpo/residual_abs": 0.17682570219039917,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.951099629193366e-08,
|
|
"logits/chosen": -2.1384809017181396,
|
|
"logits/rejected": -2.107126235961914,
|
|
"logps/chosen": -0.278969943523407,
|
|
"logps/rejected": -0.27125102281570435,
|
|
"loss": 0.3765016496181488,
|
|
"loss/core": 0.3697974681854248,
|
|
"loss/preference_sft": 0.278969943523407,
|
|
"loss/reference_anchor": 0.3073136508464813,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.84492826461792,
|
|
"rewards/margins": 1.7366445064544678,
|
|
"rewards/rejected": 1.1082838773727417,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3657851219177246,
|
|
"drift/rejected_abs": 0.19124892354011536,
|
|
"epoch": 0.7253638749701742,
|
|
"grad_norm": 7.3125,
|
|
"ht_mnpo/logit": 0.1753065586090088,
|
|
"ht_mnpo/residual": 0.17030654847621918,
|
|
"ht_mnpo/residual_abs": 0.22203508019447327,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.823044480243431e-08,
|
|
"logits/chosen": -2.1926791667938232,
|
|
"logits/rejected": -2.1352715492248535,
|
|
"logps/chosen": -0.26466232538223267,
|
|
"logps/rejected": -0.26116836071014404,
|
|
"loss": 0.4025249481201172,
|
|
"loss/core": 0.39287105202674866,
|
|
"loss/preference_sft": 0.26466232538223267,
|
|
"loss/reference_anchor": 0.45622697472572327,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.656014919281006,
|
|
"rewards/margins": 1.7530654668807983,
|
|
"rewards/rejected": 1.902949333190918,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1369520127773285,
|
|
"drift/rejected_abs": 0.0672479048371315,
|
|
"epoch": 0.7301360057265569,
|
|
"grad_norm": 19.375,
|
|
"ht_mnpo/logit": 0.07037768512964249,
|
|
"ht_mnpo/residual": 0.06537768989801407,
|
|
"ht_mnpo/residual_abs": 0.10524247586727142,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.699004497608994e-08,
|
|
"logits/chosen": -2.1493887901306152,
|
|
"logits/rejected": -2.1789956092834473,
|
|
"logps/chosen": -0.28642234206199646,
|
|
"logps/rejected": -0.29152998328208923,
|
|
"loss": 0.0944807231426239,
|
|
"loss/core": 0.09230095148086548,
|
|
"loss/preference_sft": 0.28642234206199646,
|
|
"loss/reference_anchor": 0.08034622669219971,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.368091106414795,
|
|
"rewards/margins": 0.7037767767906189,
|
|
"rewards/rejected": 0.6643142700195312,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2831539213657379,
|
|
"drift/rejected_abs": 0.2444659173488617,
|
|
"epoch": 0.7349081364829396,
|
|
"grad_norm": 72.0,
|
|
"ht_mnpo/logit": 0.03880638629198074,
|
|
"ht_mnpo/residual": 0.033806391060352325,
|
|
"ht_mnpo/residual_abs": 0.24216315150260925,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.5790263276546658e-08,
|
|
"logits/chosen": -2.191681385040283,
|
|
"logits/rejected": -2.0962822437286377,
|
|
"logps/chosen": -0.27692681550979614,
|
|
"logps/rejected": -0.25870591402053833,
|
|
"loss": 0.33768293261528015,
|
|
"loss/core": 0.32926493883132935,
|
|
"loss/preference_sft": 0.27692681550979614,
|
|
"loss/reference_anchor": 0.3932059407234192,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8315391540527344,
|
|
"rewards/margins": 0.38806402683258057,
|
|
"rewards/rejected": 2.4434750080108643,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2792401909828186,
|
|
"drift/rejected_abs": 0.08451013267040253,
|
|
"epoch": 0.7396802672393223,
|
|
"grad_norm": 7.71875,
|
|
"ht_mnpo/logit": 0.19487017393112183,
|
|
"ht_mnpo/residual": 0.18987016379833221,
|
|
"ht_mnpo/residual_abs": 0.21923527121543884,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.4631550892634126e-08,
|
|
"logits/chosen": -2.0256760120391846,
|
|
"logits/rejected": -2.054231882095337,
|
|
"logps/chosen": -0.31334948539733887,
|
|
"logps/rejected": -0.2905244827270508,
|
|
"loss": 0.7219112515449524,
|
|
"loss/core": 0.7123136520385742,
|
|
"loss/preference_sft": 0.31334948539733887,
|
|
"loss/reference_anchor": 0.4485507607460022,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.791043281555176,
|
|
"rewards/margins": 1.9487018585205078,
|
|
"rewards/rejected": 0.8423415422439575,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20807400345802307,
|
|
"drift/rejected_abs": 0.12098662555217743,
|
|
"epoch": 0.7444523979957051,
|
|
"grad_norm": 1.2890625,
|
|
"ht_mnpo/logit": 0.08689337223768234,
|
|
"ht_mnpo/residual": 0.08189336955547333,
|
|
"ht_mnpo/residual_abs": 0.08538765460252762,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.3514343568692132e-08,
|
|
"logits/chosen": -2.0675227642059326,
|
|
"logits/rejected": -2.1441054344177246,
|
|
"logps/chosen": -0.28695350885391235,
|
|
"logps/rejected": -0.2682899236679077,
|
|
"loss": 0.037697963416576385,
|
|
"loss/core": 0.03433579206466675,
|
|
"loss/preference_sft": 0.28695350885391235,
|
|
"loss/reference_anchor": 0.13941338658332825,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0788002014160156,
|
|
"rewards/margins": 0.8689336776733398,
|
|
"rewards/rejected": 1.2098662853240967,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2656870484352112,
|
|
"drift/rejected_abs": 0.13376986980438232,
|
|
"epoch": 0.7492245287520878,
|
|
"grad_norm": 604.0,
|
|
"ht_mnpo/logit": 0.1322033405303955,
|
|
"ht_mnpo/residual": 0.1272033154964447,
|
|
"ht_mnpo/residual_abs": 0.2020864486694336,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.2439061440704724e-08,
|
|
"logits/chosen": -2.1118531227111816,
|
|
"logits/rejected": -2.0152149200439453,
|
|
"logps/chosen": -0.2860553562641144,
|
|
"logps/rejected": -0.3056414723396301,
|
|
"loss": 0.2933505177497864,
|
|
"loss/core": 0.28818944096565247,
|
|
"loss/preference_sft": 0.2860553562641144,
|
|
"loss/reference_anchor": 0.22944936156272888,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6555895805358887,
|
|
"rewards/margins": 1.322033166885376,
|
|
"rewards/rejected": 1.3335561752319336,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23867455124855042,
|
|
"drift/rejected_abs": 0.11441533267498016,
|
|
"epoch": 0.7539966595084705,
|
|
"grad_norm": 14.25,
|
|
"ht_mnpo/logit": 0.12559519708156586,
|
|
"ht_mnpo/residual": 0.12059519439935684,
|
|
"ht_mnpo/residual_abs": 0.14111679792404175,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.1406108878304468e-08,
|
|
"logits/chosen": -1.9224998950958252,
|
|
"logits/rejected": -1.9974676370620728,
|
|
"logps/chosen": -0.30875641107559204,
|
|
"logps/rejected": -0.2942540645599365,
|
|
"loss": 0.2603028118610382,
|
|
"loss/core": 0.2551446557044983,
|
|
"loss/preference_sft": 0.30875641107559204,
|
|
"loss/reference_anchor": 0.2270287722349167,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3846070766448975,
|
|
"rewards/margins": 1.2559518814086914,
|
|
"rewards/rejected": 1.128654956817627,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4156590402126312,
|
|
"drift/rejected_abs": 0.2223910391330719,
|
|
"epoch": 0.7587687902648532,
|
|
"grad_norm": 152.0,
|
|
"ht_mnpo/logit": 0.1967443972826004,
|
|
"ht_mnpo/residual": 0.19174440205097198,
|
|
"ht_mnpo/residual_abs": 0.25113195180892944,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.0415874332705381e-08,
|
|
"logits/chosen": -1.9679896831512451,
|
|
"logits/rejected": -1.9452831745147705,
|
|
"logps/chosen": -0.32800760865211487,
|
|
"logps/rejected": -0.32261085510253906,
|
|
"loss": 0.5044857263565063,
|
|
"loss/core": 0.4869937300682068,
|
|
"loss/preference_sft": 0.32800760865211487,
|
|
"loss/reference_anchor": 0.8417966961860657,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 4.15133810043335,
|
|
"rewards/margins": 1.9674440622329712,
|
|
"rewards/rejected": 2.183894157409668,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26482954621315,
|
|
"drift/rejected_abs": 0.22209832072257996,
|
|
"epoch": 0.7635409210212359,
|
|
"grad_norm": 62.5,
|
|
"ht_mnpo/logit": 0.04312277212738991,
|
|
"ht_mnpo/residual": 0.038122765719890594,
|
|
"ht_mnpo/residual_abs": 0.19203929603099823,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.468730190622484e-09,
|
|
"logits/chosen": -2.1530609130859375,
|
|
"logits/rejected": -2.206618070602417,
|
|
"logps/chosen": -0.29416975378990173,
|
|
"logps/rejected": -0.2798725664615631,
|
|
"loss": 0.5578433275222778,
|
|
"loss/core": 0.5430841445922852,
|
|
"loss/preference_sft": 0.29416975378990173,
|
|
"loss/reference_anchor": 0.70854252576828,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6482741832733154,
|
|
"rewards/margins": 0.43122774362564087,
|
|
"rewards/rejected": 2.2170467376708984,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3293476700782776,
|
|
"drift/rejected_abs": 0.13683730363845825,
|
|
"epoch": 0.7683130517776187,
|
|
"grad_norm": 64.0,
|
|
"ht_mnpo/logit": 0.19290654361248016,
|
|
"ht_mnpo/residual": 0.18790654838085175,
|
|
"ht_mnpo/residual_abs": 0.21803894639015198,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 8.565032634232194e-09,
|
|
"logits/chosen": -2.0299782752990723,
|
|
"logits/rejected": -2.1024813652038574,
|
|
"logps/chosen": -0.2577776312828064,
|
|
"logps/rejected": -0.2615029811859131,
|
|
"loss": 0.2823915481567383,
|
|
"loss/core": 0.2752048373222351,
|
|
"loss/preference_sft": 0.2577776312828064,
|
|
"loss/reference_anchor": 0.3335556983947754,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.292163133621216,
|
|
"rewards/margins": 1.929065465927124,
|
|
"rewards/rejected": 1.3630976676940918,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20250026881694794,
|
|
"drift/rejected_abs": 0.09937586635351181,
|
|
"epoch": 0.7730851825340014,
|
|
"grad_norm": 7.71875,
|
|
"ht_mnpo/logit": 0.10316647589206696,
|
|
"ht_mnpo/residual": 0.09816648066043854,
|
|
"ht_mnpo/residual_abs": 0.10210967063903809,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 7.70512150722702e-09,
|
|
"logits/chosen": -2.126025676727295,
|
|
"logits/rejected": -2.2384276390075684,
|
|
"logps/chosen": -0.2960754334926605,
|
|
"logps/rejected": -0.2822602093219757,
|
|
"loss": 0.067779041826725,
|
|
"loss/core": 0.06462538242340088,
|
|
"loss/preference_sft": 0.2960754334926605,
|
|
"loss/reference_anchor": 0.12807531654834747,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0241200923919678,
|
|
"rewards/margins": 1.0316648483276367,
|
|
"rewards/rejected": 0.9924553632736206,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28037869930267334,
|
|
"drift/rejected_abs": 0.09076918661594391,
|
|
"epoch": 0.7778573132903842,
|
|
"grad_norm": 11.5625,
|
|
"ht_mnpo/logit": 0.18967393040657043,
|
|
"ht_mnpo/residual": 0.18467390537261963,
|
|
"ht_mnpo/residual_abs": 0.18819065392017365,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.8893201870139915e-09,
|
|
"logits/chosen": -2.2424092292785645,
|
|
"logits/rejected": -2.2913436889648438,
|
|
"logps/chosen": -0.2803761065006256,
|
|
"logps/rejected": -0.2700513005256653,
|
|
"loss": 0.47603359818458557,
|
|
"loss/core": 0.46868079900741577,
|
|
"loss/preference_sft": 0.2803761065006256,
|
|
"loss/reference_anchor": 0.33960336446762085,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8023884296417236,
|
|
"rewards/margins": 1.8967387676239014,
|
|
"rewards/rejected": 0.9056491851806641,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37526756525039673,
|
|
"drift/rejected_abs": 0.11981121450662613,
|
|
"epoch": 0.7826294440467669,
|
|
"grad_norm": 0.734375,
|
|
"ht_mnpo/logit": 0.25736114382743835,
|
|
"ht_mnpo/residual": 0.2523611783981323,
|
|
"ht_mnpo/residual_abs": 0.2597852051258087,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.117935463105808e-09,
|
|
"logits/chosen": -2.123929500579834,
|
|
"logits/rejected": -2.1171352863311768,
|
|
"logps/chosen": -0.2949315905570984,
|
|
"logps/rejected": -0.30626291036605835,
|
|
"loss": 1.008721113204956,
|
|
"loss/core": 0.9946671724319458,
|
|
"loss/preference_sft": 0.2949315905570984,
|
|
"loss/reference_anchor": 0.6732128858566284,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.751708507537842,
|
|
"rewards/margins": 2.5736114978790283,
|
|
"rewards/rejected": 1.178097128868103,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2473944127559662,
|
|
"drift/rejected_abs": 0.12352682650089264,
|
|
"epoch": 0.7874015748031497,
|
|
"grad_norm": 27.75,
|
|
"ht_mnpo/logit": 0.12414328753948212,
|
|
"ht_mnpo/residual": 0.11914326995611191,
|
|
"ht_mnpo/residual_abs": 0.18271097540855408,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 5.391257421749826e-09,
|
|
"logits/chosen": -2.0234365463256836,
|
|
"logits/rejected": -2.0557427406311035,
|
|
"logps/chosen": -0.27447009086608887,
|
|
"logps/rejected": -0.26940232515335083,
|
|
"loss": 0.41700777411460876,
|
|
"loss/core": 0.4105207026004791,
|
|
"loss/preference_sft": 0.27447009086608887,
|
|
"loss/reference_anchor": 0.2969074547290802,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4735748767852783,
|
|
"rewards/margins": 1.241432785987854,
|
|
"rewards/rejected": 1.2321420907974243,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2917875647544861,
|
|
"drift/rejected_abs": 0.17095866799354553,
|
|
"epoch": 0.7921737055595324,
|
|
"grad_norm": 224.0,
|
|
"ht_mnpo/logit": 0.1259477138519287,
|
|
"ht_mnpo/residual": 0.1209477037191391,
|
|
"ht_mnpo/residual_abs": 0.19740599393844604,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.709559336838462e-09,
|
|
"logits/chosen": -2.0139071941375732,
|
|
"logits/rejected": -2.0471396446228027,
|
|
"logps/chosen": -0.2997593581676483,
|
|
"logps/rejected": -0.28049224615097046,
|
|
"loss": 0.37333980202674866,
|
|
"loss/core": 0.354358971118927,
|
|
"loss/preference_sft": 0.2997593581676483,
|
|
"loss/reference_anchor": 0.9190651774406433,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.915761947631836,
|
|
"rewards/margins": 1.259477138519287,
|
|
"rewards/rejected": 1.6562846899032593,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2739551067352295,
|
|
"drift/rejected_abs": 0.15794166922569275,
|
|
"epoch": 0.7969458363159151,
|
|
"grad_norm": 6.71875,
|
|
"ht_mnpo/logit": 0.11599205434322357,
|
|
"ht_mnpo/residual": 0.11099205911159515,
|
|
"ht_mnpo/residual_abs": 0.16929420828819275,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.073097567142025e-09,
|
|
"logits/chosen": -1.9657586812973022,
|
|
"logits/rejected": -1.9134137630462646,
|
|
"logps/chosen": -0.32205599546432495,
|
|
"logps/rejected": -0.27461498975753784,
|
|
"loss": 0.4663979411125183,
|
|
"loss/core": 0.45191946625709534,
|
|
"loss/preference_sft": 0.32205599546432495,
|
|
"loss/reference_anchor": 0.6917184591293335,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.7391560077667236,
|
|
"rewards/margins": 1.159920573234558,
|
|
"rewards/rejected": 1.5792356729507446,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17836840450763702,
|
|
"drift/rejected_abs": 0.25512903928756714,
|
|
"epoch": 0.8017179670722978,
|
|
"grad_norm": 476.0,
|
|
"ht_mnpo/logit": -0.07672592997550964,
|
|
"ht_mnpo/residual": -0.08172592520713806,
|
|
"ht_mnpo/residual_abs": 0.20435388386249542,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.482111459902695e-09,
|
|
"logits/chosen": -2.216046094894409,
|
|
"logits/rejected": -2.06156325340271,
|
|
"logps/chosen": -0.28472912311553955,
|
|
"logps/rejected": -0.2904352843761444,
|
|
"loss": 0.6602007746696472,
|
|
"loss/core": 0.6512366533279419,
|
|
"loss/preference_sft": 0.28472912311553955,
|
|
"loss/reference_anchor": 0.4197351336479187,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.781996726989746,
|
|
"rewards/margins": -0.7672592997550964,
|
|
"rewards/rejected": 2.549255847930908,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2331835776567459,
|
|
"drift/rejected_abs": 0.17626038193702698,
|
|
"epoch": 0.8064900978286805,
|
|
"grad_norm": 3.984375,
|
|
"ht_mnpo/logit": 0.056995175778865814,
|
|
"ht_mnpo/residual": 0.051995180547237396,
|
|
"ht_mnpo/residual_abs": 0.18106596171855927,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.9368232608258797e-09,
|
|
"logits/chosen": -2.004657506942749,
|
|
"logits/rejected": -2.017953872680664,
|
|
"logps/chosen": -0.30708542466163635,
|
|
"logps/rejected": -0.31051960587501526,
|
|
"loss": 0.3166196346282959,
|
|
"loss/core": 0.31019559502601624,
|
|
"loss/preference_sft": 0.30708542466163635,
|
|
"loss/reference_anchor": 0.2904927730560303,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3293375968933105,
|
|
"rewards/margins": 0.5699518322944641,
|
|
"rewards/rejected": 1.7593857049942017,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2094683200120926,
|
|
"drift/rejected_abs": 0.10852241516113281,
|
|
"epoch": 0.8112622285850632,
|
|
"grad_norm": 1.4140625,
|
|
"ht_mnpo/logit": 0.10089180618524551,
|
|
"ht_mnpo/residual": 0.0958918035030365,
|
|
"ht_mnpo/residual_abs": 0.09958347678184509,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.4374380305025866e-09,
|
|
"logits/chosen": -2.041243314743042,
|
|
"logits/rejected": -2.049227714538574,
|
|
"logps/chosen": -0.2886284291744232,
|
|
"logps/rejected": -0.27855145931243896,
|
|
"loss": 0.08112355321645737,
|
|
"loss/core": 0.07786046713590622,
|
|
"loss/preference_sft": 0.2886284291744232,
|
|
"loss/reference_anchor": 0.13429181277751923,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0938479900360107,
|
|
"rewards/margins": 1.008918046951294,
|
|
"rewards/rejected": 1.084930181503296,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16252204775810242,
|
|
"drift/rejected_abs": 0.1045575961470604,
|
|
"epoch": 0.816034359341446,
|
|
"grad_norm": 1.0390625,
|
|
"ht_mnpo/logit": 0.05791337043046951,
|
|
"ht_mnpo/residual": 0.0529133677482605,
|
|
"ht_mnpo/residual_abs": 0.07363729178905487,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.984143567294594e-09,
|
|
"logits/chosen": -2.1060547828674316,
|
|
"logits/rejected": -2.0476009845733643,
|
|
"logps/chosen": -0.29853376746177673,
|
|
"logps/rejected": -0.2876315116882324,
|
|
"loss": 0.03223501145839691,
|
|
"loss/core": 0.029459426179528236,
|
|
"loss/preference_sft": 0.29853376746177673,
|
|
"loss/reference_anchor": 0.10892575979232788,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.623846411705017,
|
|
"rewards/margins": 0.5791337490081787,
|
|
"rewards/rejected": 1.044712781906128,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3076639473438263,
|
|
"drift/rejected_abs": 0.15537162125110626,
|
|
"epoch": 0.8208064900978287,
|
|
"grad_norm": 16.375,
|
|
"ht_mnpo/logit": 0.1523175835609436,
|
|
"ht_mnpo/residual": 0.147317573428154,
|
|
"ht_mnpo/residual_abs": 0.1500585526227951,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.577110336711096e-09,
|
|
"logits/chosen": -2.2431137561798096,
|
|
"logits/rejected": -2.4290809631347656,
|
|
"logps/chosen": -0.2704775035381317,
|
|
"logps/rejected": -0.2497967928647995,
|
|
"loss": 0.26532191038131714,
|
|
"loss/core": 0.25764161348342896,
|
|
"loss/preference_sft": 0.2704775035381317,
|
|
"loss/reference_anchor": 0.3569665551185608,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0762779712677,
|
|
"rewards/margins": 1.523175835609436,
|
|
"rewards/rejected": 1.5531020164489746,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17823238670825958,
|
|
"drift/rejected_abs": 0.09530796110630035,
|
|
"epoch": 0.8255786208542114,
|
|
"grad_norm": 1.5859375,
|
|
"ht_mnpo/logit": 0.08302712440490723,
|
|
"ht_mnpo/residual": 0.07802711427211761,
|
|
"ht_mnpo/residual_abs": 0.10461542755365372,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.2164914073037048e-09,
|
|
"logits/chosen": -2.0242111682891846,
|
|
"logits/rejected": -2.03706693649292,
|
|
"logps/chosen": -0.32036247849464417,
|
|
"logps/rejected": -0.2977069318294525,
|
|
"loss": 0.25833559036254883,
|
|
"loss/core": 0.2516372799873352,
|
|
"loss/preference_sft": 0.32036247849464417,
|
|
"loss/reference_anchor": 0.3028794229030609,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7823235988616943,
|
|
"rewards/margins": 0.8302712440490723,
|
|
"rewards/rejected": 0.9520524740219116,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21682997047901154,
|
|
"drift/rejected_abs": 0.20350849628448486,
|
|
"epoch": 0.8303507516105941,
|
|
"grad_norm": 972.0,
|
|
"ht_mnpo/logit": 0.013434367254376411,
|
|
"ht_mnpo/residual": 0.008434370160102844,
|
|
"ht_mnpo/residual_abs": 0.1993631273508072,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.024223931035357e-10,
|
|
"logits/chosen": -2.161740779876709,
|
|
"logits/rejected": -2.1781668663024902,
|
|
"logps/chosen": -0.24697892367839813,
|
|
"logps/rejected": -0.2532227039337158,
|
|
"loss": 0.6162554025650024,
|
|
"loss/core": 0.6080134510993958,
|
|
"loss/preference_sft": 0.24697892367839813,
|
|
"loss/reference_anchor": 0.3873947560787201,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.1679275035858154,
|
|
"rewards/margins": 0.13434359431266785,
|
|
"rewards/rejected": 2.033583641052246,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2778306007385254,
|
|
"drift/rejected_abs": 0.1163145899772644,
|
|
"epoch": 0.8351228823669768,
|
|
"grad_norm": 1616.0,
|
|
"ht_mnpo/logit": 0.17591026425361633,
|
|
"ht_mnpo/residual": 0.17091026902198792,
|
|
"ht_mnpo/residual_abs": 0.23106594383716583,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 6.350214026223516e-10,
|
|
"logits/chosen": -2.0502777099609375,
|
|
"logits/rejected": -2.0214967727661133,
|
|
"logps/chosen": -0.27098390460014343,
|
|
"logps/rejected": -0.2793070077896118,
|
|
"loss": 0.7463462352752686,
|
|
"loss/core": 0.7381068468093872,
|
|
"loss/preference_sft": 0.27098390460014343,
|
|
"loss/reference_anchor": 0.3848689794540405,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.7779488563537598,
|
|
"rewards/margins": 1.7591025829315186,
|
|
"rewards/rejected": 1.0188462734222412,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12231247127056122,
|
|
"drift/rejected_abs": 0.07507012039422989,
|
|
"epoch": 0.8398950131233596,
|
|
"grad_norm": 2.71875,
|
|
"ht_mnpo/logit": 0.04689433425664902,
|
|
"ht_mnpo/residual": 0.0418943352997303,
|
|
"ht_mnpo/residual_abs": 0.051383644342422485,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 4.143889944367429e-10,
|
|
"logits/chosen": -2.061795711517334,
|
|
"logits/rejected": -2.1499526500701904,
|
|
"logps/chosen": -0.27764564752578735,
|
|
"logps/rejected": -0.27403324842453003,
|
|
"loss": 0.010318822227418423,
|
|
"loss/core": 0.009077044203877449,
|
|
"loss/preference_sft": 0.27764564752578735,
|
|
"loss/reference_anchor": 0.0343242809176445,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2161328792572021,
|
|
"rewards/margins": 0.4689432978630066,
|
|
"rewards/rejected": 0.7471895217895508,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14238771796226501,
|
|
"drift/rejected_abs": 0.06504310667514801,
|
|
"epoch": 0.8446671438797423,
|
|
"grad_norm": 1.9609375,
|
|
"ht_mnpo/logit": 0.07748010009527206,
|
|
"ht_mnpo/residual": 0.07248009741306305,
|
|
"ht_mnpo/residual_abs": 0.08879692852497101,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 2.406081393722531e-10,
|
|
"logits/chosen": -2.0421433448791504,
|
|
"logits/rejected": -2.0938682556152344,
|
|
"logps/chosen": -0.30008670687675476,
|
|
"logps/rejected": -0.28426411747932434,
|
|
"loss": 0.0430099293589592,
|
|
"loss/core": 0.04152857884764671,
|
|
"loss/preference_sft": 0.30008670687675476,
|
|
"loss/reference_anchor": 0.04405852407217026,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4236747026443481,
|
|
"rewards/margins": 0.7748010754585266,
|
|
"rewards/rejected": 0.6488736867904663,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34237349033355713,
|
|
"drift/rejected_abs": 0.1923760622739792,
|
|
"epoch": 0.849439274636125,
|
|
"grad_norm": 624.0,
|
|
"ht_mnpo/logit": 0.15943066775798798,
|
|
"ht_mnpo/residual": 0.15443067252635956,
|
|
"ht_mnpo/residual_abs": 0.2659396827220917,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 1.1374418930135133e-10,
|
|
"logits/chosen": -2.10282564163208,
|
|
"logits/rejected": -2.1714038848876953,
|
|
"logps/chosen": -0.2946470379829407,
|
|
"logps/rejected": -0.3199237585067749,
|
|
"loss": 0.8831459283828735,
|
|
"loss/core": 0.8672249913215637,
|
|
"loss/preference_sft": 0.2946470379829407,
|
|
"loss/reference_anchor": 0.766579806804657,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.4213645458221436,
|
|
"rewards/margins": 1.5943067073822021,
|
|
"rewards/rejected": 1.8270580768585205,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2455691546201706,
|
|
"drift/rejected_abs": 0.10265467315912247,
|
|
"epoch": 0.8542114053925077,
|
|
"grad_norm": 17.75,
|
|
"ht_mnpo/logit": 0.1426907479763031,
|
|
"ht_mnpo/residual": 0.1376907378435135,
|
|
"ht_mnpo/residual_abs": 0.1457034945487976,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 3.3844852567285756e-11,
|
|
"logits/chosen": -2.1894445419311523,
|
|
"logits/rejected": -2.145141124725342,
|
|
"logps/chosen": -0.3019161522388458,
|
|
"logps/rejected": -0.2727174162864685,
|
|
"loss": 0.39371412992477417,
|
|
"loss/core": 0.3860507309436798,
|
|
"loss/preference_sft": 0.3019161522388458,
|
|
"loss/reference_anchor": 0.3529819846153259,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.4513144493103027,
|
|
"rewards/margins": 1.4269073009490967,
|
|
"rewards/rejected": 1.024407148361206,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.253998339176178,
|
|
"drift/rejected_abs": 0.14686278998851776,
|
|
"epoch": 0.8589835361488904,
|
|
"grad_norm": 0.27734375,
|
|
"ht_mnpo/logit": 0.10787560790777206,
|
|
"ht_mnpo/residual": 0.10287560522556305,
|
|
"ht_mnpo/residual_abs": 0.10665620863437653,
|
|
"ht_mnpo/target": 0.004999999888241291,
|
|
"ht_mnpo/target_abs": 0.004999999888241291,
|
|
"learning_rate": 9.401760429905703e-13,
|
|
"logits/chosen": -2.1683638095855713,
|
|
"logits/rejected": -2.2042431831359863,
|
|
"logps/chosen": -0.3270532488822937,
|
|
"logps/rejected": -0.30916860699653625,
|
|
"loss": 0.22434833645820618,
|
|
"loss/core": 0.21211738884449005,
|
|
"loss/preference_sft": 0.3270532488822937,
|
|
"loss/reference_anchor": 0.5788421034812927,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5390026569366455,
|
|
"rewards/margins": 1.0787560939788818,
|
|
"rewards/rejected": 1.4602468013763428,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8589835361488904,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.36845981503112446,
|
|
"train_runtime": 7265.877,
|
|
"train_samples_per_second": 1.982,
|
|
"train_steps_per_second": 0.124
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|