0738f2ca0a
Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s42 Source: Original Platform
4544 lines
178 KiB
JSON
4544 lines
178 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8600609209819029,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.2093760073184967,
|
|
"drift/rejected_abs": 0.1779964119195938,
|
|
"epoch": 0.004778116227677238,
|
|
"grad_norm": 608.0,
|
|
"ht_mnpo/logit": 0.03132759779691696,
|
|
"ht_mnpo/residual": 0.02382758818566799,
|
|
"ht_mnpo/residual_abs": 0.24814370274543762,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.111111111111111e-08,
|
|
"logits/chosen": -2.080493211746216,
|
|
"logits/rejected": -1.8775554895401,
|
|
"logps/chosen": -0.31729820370674133,
|
|
"logps/rejected": -0.32572638988494873,
|
|
"loss": 0.8218833208084106,
|
|
"loss/core": 0.8120266199111938,
|
|
"loss/preference_sft": 0.31729820370674133,
|
|
"loss/reference_anchor": 0.46110573410987854,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0926239490509033,
|
|
"rewards/margins": 0.3132759928703308,
|
|
"rewards/rejected": 1.7793481349945068,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26713913679122925,
|
|
"drift/rejected_abs": 0.12592381238937378,
|
|
"epoch": 0.009556232455354476,
|
|
"grad_norm": 5.5625,
|
|
"ht_mnpo/logit": 0.14135970175266266,
|
|
"ht_mnpo/residual": 0.13385972380638123,
|
|
"ht_mnpo/residual_abs": 0.14800478518009186,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5e-08,
|
|
"logits/chosen": -2.263683557510376,
|
|
"logits/rejected": -2.1684491634368896,
|
|
"logps/chosen": -0.30673515796661377,
|
|
"logps/rejected": -0.3016442656517029,
|
|
"loss": 0.32831090688705444,
|
|
"loss/core": 0.31749722361564636,
|
|
"loss/preference_sft": 0.30673515796661377,
|
|
"loss/reference_anchor": 0.5100095868110657,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6701548099517822,
|
|
"rewards/margins": 1.4135972261428833,
|
|
"rewards/rejected": 1.2565577030181885,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22741301357746124,
|
|
"drift/rejected_abs": 0.07075328379869461,
|
|
"epoch": 0.014334348683031715,
|
|
"grad_norm": 1616.0,
|
|
"ht_mnpo/logit": 0.1659075915813446,
|
|
"ht_mnpo/residual": 0.15840759873390198,
|
|
"ht_mnpo/residual_abs": 0.16668769717216492,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.888888888888889e-08,
|
|
"logits/chosen": -2.327167272567749,
|
|
"logits/rejected": -2.2666492462158203,
|
|
"logps/chosen": -0.30080440640449524,
|
|
"logps/rejected": -0.28799667954444885,
|
|
"loss": 0.6700844764709473,
|
|
"loss/core": 0.6620814204216003,
|
|
"loss/preference_sft": 0.30080440640449524,
|
|
"loss/reference_anchor": 0.3700721263885498,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2735326290130615,
|
|
"rewards/margins": 1.6590759754180908,
|
|
"rewards/rejected": 0.6144567131996155,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.236248180270195,
|
|
"drift/rejected_abs": 0.2190016508102417,
|
|
"epoch": 0.019112464910708952,
|
|
"grad_norm": 176.0,
|
|
"ht_mnpo/logit": 0.01763411983847618,
|
|
"ht_mnpo/residual": 0.01013412605971098,
|
|
"ht_mnpo/residual_abs": 0.1537032574415207,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.2777777777777776e-08,
|
|
"logits/chosen": -2.3429882526397705,
|
|
"logits/rejected": -2.1697194576263428,
|
|
"logps/chosen": -0.2879870533943176,
|
|
"logps/rejected": -0.30968016386032104,
|
|
"loss": 0.3623378872871399,
|
|
"loss/core": 0.3473840057849884,
|
|
"loss/preference_sft": 0.2879870533943176,
|
|
"loss/reference_anchor": 0.7188963890075684,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.3624815940856934,
|
|
"rewards/margins": 0.17634114623069763,
|
|
"rewards/rejected": 2.186140537261963,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29826819896698,
|
|
"drift/rejected_abs": 0.179667666554451,
|
|
"epoch": 0.023890581138386192,
|
|
"grad_norm": 6.3125,
|
|
"ht_mnpo/logit": 0.11871922016143799,
|
|
"ht_mnpo/residual": 0.11121921241283417,
|
|
"ht_mnpo/residual_abs": 0.1725720465183258,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.666666666666667e-08,
|
|
"logits/chosen": -2.0951151847839355,
|
|
"logits/rejected": -1.8950538635253906,
|
|
"logps/chosen": -0.29473376274108887,
|
|
"logps/rejected": -0.30570217967033386,
|
|
"loss": 0.32861581444740295,
|
|
"loss/core": 0.3204094469547272,
|
|
"loss/preference_sft": 0.29473376274108887,
|
|
"loss/reference_anchor": 0.3808436989784241,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.981147050857544,
|
|
"rewards/margins": 1.1871918439865112,
|
|
"rewards/rejected": 1.7939550876617432,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15562030673027039,
|
|
"drift/rejected_abs": 0.09274661540985107,
|
|
"epoch": 0.02866869736606343,
|
|
"grad_norm": 18.0,
|
|
"ht_mnpo/logit": 0.06269876658916473,
|
|
"ht_mnpo/residual": 0.05519877001643181,
|
|
"ht_mnpo/residual_abs": 0.08922450989484787,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.055555555555555e-08,
|
|
"logits/chosen": -2.2576093673706055,
|
|
"logits/rejected": -2.082545042037964,
|
|
"logps/chosen": -0.29427456855773926,
|
|
"logps/rejected": -0.2952222228050232,
|
|
"loss": 0.07133828848600388,
|
|
"loss/core": 0.06886543333530426,
|
|
"loss/preference_sft": 0.29427456855773926,
|
|
"loss/reference_anchor": 0.09421543031930923,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.5543291568756104,
|
|
"rewards/margins": 0.6269876956939697,
|
|
"rewards/rejected": 0.9273414611816406,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27138715982437134,
|
|
"drift/rejected_abs": 0.3082945942878723,
|
|
"epoch": 0.03344681359374067,
|
|
"grad_norm": 58.0,
|
|
"ht_mnpo/logit": -0.03644378110766411,
|
|
"ht_mnpo/residual": -0.043943800032138824,
|
|
"ht_mnpo/residual_abs": 0.25199875235557556,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.444444444444444e-08,
|
|
"logits/chosen": -2.179772138595581,
|
|
"logits/rejected": -1.8775641918182373,
|
|
"logps/chosen": -0.2725084125995636,
|
|
"logps/rejected": -0.30440372228622437,
|
|
"loss": 0.5291619300842285,
|
|
"loss/core": 0.5189464092254639,
|
|
"loss/preference_sft": 0.2725084125995636,
|
|
"loss/reference_anchor": 0.4835292398929596,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.712721824645996,
|
|
"rewards/margins": -0.3644378185272217,
|
|
"rewards/rejected": 3.0771596431732178,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25635233521461487,
|
|
"drift/rejected_abs": 0.1467134654521942,
|
|
"epoch": 0.038224929821417904,
|
|
"grad_norm": 1.7421875,
|
|
"ht_mnpo/logit": 0.12074834108352661,
|
|
"ht_mnpo/residual": 0.11324834823608398,
|
|
"ht_mnpo/residual_abs": 0.17528986930847168,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0833333333333334e-07,
|
|
"logits/chosen": -2.024980068206787,
|
|
"logits/rejected": -1.7612985372543335,
|
|
"logps/chosen": -0.2727901041507721,
|
|
"logps/rejected": -0.27953046560287476,
|
|
"loss": 0.46636730432510376,
|
|
"loss/core": 0.4604361057281494,
|
|
"loss/preference_sft": 0.2727901041507721,
|
|
"loss/reference_anchor": 0.2692839503288269,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.563462495803833,
|
|
"rewards/margins": 1.2074834108352661,
|
|
"rewards/rejected": 1.3559788465499878,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3089280426502228,
|
|
"drift/rejected_abs": 0.1763952225446701,
|
|
"epoch": 0.04300304604909514,
|
|
"grad_norm": 0.353515625,
|
|
"ht_mnpo/logit": 0.13275332748889923,
|
|
"ht_mnpo/residual": 0.1252533495426178,
|
|
"ht_mnpo/residual_abs": 0.17680028080940247,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2222222222222222e-07,
|
|
"logits/chosen": -2.1491565704345703,
|
|
"logits/rejected": -1.917433738708496,
|
|
"logps/chosen": -0.28032320737838745,
|
|
"logps/rejected": -0.28377169370651245,
|
|
"loss": 0.4325074553489685,
|
|
"loss/core": 0.41803789138793945,
|
|
"loss/preference_sft": 0.28032320737838745,
|
|
"loss/reference_anchor": 0.6954492330551147,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.088975429534912,
|
|
"rewards/margins": 1.3275333642959595,
|
|
"rewards/rejected": 1.7614424228668213,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20034293830394745,
|
|
"drift/rejected_abs": 0.16037920117378235,
|
|
"epoch": 0.047781162276772385,
|
|
"grad_norm": 228.0,
|
|
"ht_mnpo/logit": 0.03978969529271126,
|
|
"ht_mnpo/residual": 0.03228970244526863,
|
|
"ht_mnpo/residual_abs": 0.13686040043830872,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3611111111111108e-07,
|
|
"logits/chosen": -2.39717960357666,
|
|
"logits/rejected": -2.20611572265625,
|
|
"logps/chosen": -0.27169185876846313,
|
|
"logps/rejected": -0.2966422736644745,
|
|
"loss": 0.21604546904563904,
|
|
"loss/core": 0.2099648416042328,
|
|
"loss/preference_sft": 0.27169185876846313,
|
|
"loss/reference_anchor": 0.27686262130737305,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.9987766742706299,
|
|
"rewards/margins": 0.39789706468582153,
|
|
"rewards/rejected": 1.6008796691894531,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2341303527355194,
|
|
"drift/rejected_abs": 0.16907049715518951,
|
|
"epoch": 0.05255927850444962,
|
|
"grad_norm": 136.0,
|
|
"ht_mnpo/logit": 0.06506579369306564,
|
|
"ht_mnpo/residual": 0.05756579712033272,
|
|
"ht_mnpo/residual_abs": 0.07719553261995316,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5e-07,
|
|
"logits/chosen": -2.284651279449463,
|
|
"logits/rejected": -2.0335497856140137,
|
|
"logps/chosen": -0.2873196005821228,
|
|
"logps/rejected": -0.28336891531944275,
|
|
"loss": 0.055111657828092575,
|
|
"loss/core": 0.04755812883377075,
|
|
"loss/preference_sft": 0.2873196005821228,
|
|
"loss/reference_anchor": 0.3489445447921753,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3410696983337402,
|
|
"rewards/margins": 0.6506578922271729,
|
|
"rewards/rejected": 1.6904115676879883,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17374685406684875,
|
|
"drift/rejected_abs": 0.11681034415960312,
|
|
"epoch": 0.05733739473212686,
|
|
"grad_norm": 16.625,
|
|
"ht_mnpo/logit": 0.05747275426983833,
|
|
"ht_mnpo/residual": 0.049972742795944214,
|
|
"ht_mnpo/residual_abs": 0.12492366135120392,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6388888888888888e-07,
|
|
"logits/chosen": -2.3623435497283936,
|
|
"logits/rejected": -2.1003968715667725,
|
|
"logps/chosen": -0.27126988768577576,
|
|
"logps/rejected": -0.27584511041641235,
|
|
"loss": 0.14398963749408722,
|
|
"loss/core": 0.1407667100429535,
|
|
"loss/preference_sft": 0.27126988768577576,
|
|
"loss/reference_anchor": 0.13401837646961212,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.7370201349258423,
|
|
"rewards/margins": 0.574727475643158,
|
|
"rewards/rejected": 1.162292718887329,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2348300963640213,
|
|
"drift/rejected_abs": 0.1319127380847931,
|
|
"epoch": 0.062115510959804096,
|
|
"grad_norm": 3.5625,
|
|
"ht_mnpo/logit": 0.10298211872577667,
|
|
"ht_mnpo/residual": 0.09548211842775345,
|
|
"ht_mnpo/residual_abs": 0.12007130682468414,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7777777777777776e-07,
|
|
"logits/chosen": -2.171497344970703,
|
|
"logits/rejected": -1.954334020614624,
|
|
"logps/chosen": -0.2881249189376831,
|
|
"logps/rejected": -0.2843600809574127,
|
|
"loss": 0.2871479094028473,
|
|
"loss/core": 0.2757525146007538,
|
|
"loss/preference_sft": 0.2881249189376831,
|
|
"loss/reference_anchor": 0.5409581065177917,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3482909202575684,
|
|
"rewards/margins": 1.0298211574554443,
|
|
"rewards/rejected": 1.3184694051742554,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1808304786682129,
|
|
"drift/rejected_abs": 0.11002807319164276,
|
|
"epoch": 0.06689362718748133,
|
|
"grad_norm": 71.5,
|
|
"ht_mnpo/logit": 0.07084055244922638,
|
|
"ht_mnpo/residual": 0.06334055960178375,
|
|
"ht_mnpo/residual_abs": 0.10876189172267914,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.9166666666666668e-07,
|
|
"logits/chosen": -2.284392833709717,
|
|
"logits/rejected": -2.0471653938293457,
|
|
"logps/chosen": -0.2547953724861145,
|
|
"logps/rejected": -0.2760739028453827,
|
|
"loss": 0.09047970175743103,
|
|
"loss/core": 0.08808381855487823,
|
|
"loss/preference_sft": 0.2547953724861145,
|
|
"loss/reference_anchor": 0.09431412816047668,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.8078711032867432,
|
|
"rewards/margins": 0.708405613899231,
|
|
"rewards/rejected": 1.0994654893875122,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.257373571395874,
|
|
"drift/rejected_abs": 0.20142784714698792,
|
|
"epoch": 0.07167174341515857,
|
|
"grad_norm": 19.5,
|
|
"ht_mnpo/logit": 0.055962108075618744,
|
|
"ht_mnpo/residual": 0.04846210405230522,
|
|
"ht_mnpo/residual_abs": 0.1752299815416336,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0555555555555553e-07,
|
|
"logits/chosen": -2.2204973697662354,
|
|
"logits/rejected": -1.941910743713379,
|
|
"logps/chosen": -0.2695019841194153,
|
|
"logps/rejected": -0.27252477407455444,
|
|
"loss": 0.40702590346336365,
|
|
"loss/core": 0.39896032214164734,
|
|
"loss/preference_sft": 0.2695019841194153,
|
|
"loss/reference_anchor": 0.3763294816017151,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.5731680393218994,
|
|
"rewards/margins": 0.5596210360527039,
|
|
"rewards/rejected": 2.01354718208313,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.327124685049057,
|
|
"drift/rejected_abs": 0.169361412525177,
|
|
"epoch": 0.07644985964283581,
|
|
"grad_norm": 87.0,
|
|
"ht_mnpo/logit": 0.15786626935005188,
|
|
"ht_mnpo/residual": 0.15036630630493164,
|
|
"ht_mnpo/residual_abs": 0.21834254264831543,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1944444444444442e-07,
|
|
"logits/chosen": -2.004788398742676,
|
|
"logits/rejected": -1.7699687480926514,
|
|
"logps/chosen": -0.2821227014064789,
|
|
"logps/rejected": -0.31434106826782227,
|
|
"loss": 0.5702736377716064,
|
|
"loss/core": 0.5585130453109741,
|
|
"loss/preference_sft": 0.2821227014064789,
|
|
"loss/reference_anchor": 0.5598186254501343,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.2704272270202637,
|
|
"rewards/margins": 1.578662633895874,
|
|
"rewards/rejected": 1.6917644739151,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17778277397155762,
|
|
"drift/rejected_abs": 0.11436925828456879,
|
|
"epoch": 0.08122797587051304,
|
|
"grad_norm": 0.71875,
|
|
"ht_mnpo/logit": 0.06345954537391663,
|
|
"ht_mnpo/residual": 0.0559595450758934,
|
|
"ht_mnpo/residual_abs": 0.1518561840057373,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3333333333333333e-07,
|
|
"logits/chosen": -2.423524856567383,
|
|
"logits/rejected": -2.1058201789855957,
|
|
"logps/chosen": -0.30849772691726685,
|
|
"logps/rejected": -0.3133195638656616,
|
|
"loss": 0.34925132989883423,
|
|
"loss/core": 0.34384849667549133,
|
|
"loss/preference_sft": 0.30849772691726685,
|
|
"loss/reference_anchor": 0.23929384350776672,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.7773349285125732,
|
|
"rewards/margins": 0.634595513343811,
|
|
"rewards/rejected": 1.1427395343780518,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2773793935775757,
|
|
"drift/rejected_abs": 0.21731296181678772,
|
|
"epoch": 0.08600609209819028,
|
|
"grad_norm": 6.125,
|
|
"ht_mnpo/logit": 0.0599784180521965,
|
|
"ht_mnpo/residual": 0.05247841402888298,
|
|
"ht_mnpo/residual_abs": 0.250608891248703,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4722222222222224e-07,
|
|
"logits/chosen": -2.041780471801758,
|
|
"logits/rejected": -1.831589698791504,
|
|
"logps/chosen": -0.288813978433609,
|
|
"logps/rejected": -0.2829095423221588,
|
|
"loss": 0.8480337262153625,
|
|
"loss/core": 0.8339030146598816,
|
|
"loss/preference_sft": 0.288813978433609,
|
|
"loss/reference_anchor": 0.6776567697525024,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.7713189125061035,
|
|
"rewards/margins": 0.5997841954231262,
|
|
"rewards/rejected": 2.171534299850464,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2536742091178894,
|
|
"drift/rejected_abs": 0.154718279838562,
|
|
"epoch": 0.09078420832586753,
|
|
"grad_norm": 1.2578125,
|
|
"ht_mnpo/logit": 0.09875840693712234,
|
|
"ht_mnpo/residual": 0.09125840663909912,
|
|
"ht_mnpo/residual_abs": 0.12308160215616226,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4998495746616845e-07,
|
|
"logits/chosen": -2.167048215866089,
|
|
"logits/rejected": -2.0313143730163574,
|
|
"logps/chosen": -0.2899051606655121,
|
|
"logps/rejected": -0.27663424611091614,
|
|
"loss": 0.19379271566867828,
|
|
"loss/core": 0.18530288338661194,
|
|
"loss/preference_sft": 0.2899051606655121,
|
|
"loss/reference_anchor": 0.39550065994262695,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.53287935256958,
|
|
"rewards/margins": 0.987584114074707,
|
|
"rewards/rejected": 1.5452953577041626,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23249800503253937,
|
|
"drift/rejected_abs": 0.22857382893562317,
|
|
"epoch": 0.09556232455354477,
|
|
"grad_norm": 24.75,
|
|
"ht_mnpo/logit": 0.003926327917724848,
|
|
"ht_mnpo/residual": -0.0035736740101128817,
|
|
"ht_mnpo/residual_abs": 0.19224998354911804,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4992385337738696e-07,
|
|
"logits/chosen": -2.255802631378174,
|
|
"logits/rejected": -1.9103189706802368,
|
|
"logps/chosen": -0.25300678610801697,
|
|
"logps/rejected": -0.2917603552341461,
|
|
"loss": 0.23590651154518127,
|
|
"loss/core": 0.2300805300474167,
|
|
"loss/preference_sft": 0.25300678610801697,
|
|
"loss/reference_anchor": 0.26599597930908203,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.3240771293640137,
|
|
"rewards/margins": 0.039263151586055756,
|
|
"rewards/rejected": 2.2848141193389893,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18887114524841309,
|
|
"drift/rejected_abs": 0.2856203615665436,
|
|
"epoch": 0.100340440781222,
|
|
"grad_norm": 580.0,
|
|
"ht_mnpo/logit": -0.0967492014169693,
|
|
"ht_mnpo/residual": -0.10424920171499252,
|
|
"ht_mnpo/residual_abs": 0.22890052199363708,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4981577053636144e-07,
|
|
"logits/chosen": -2.0870354175567627,
|
|
"logits/rejected": -1.7681773900985718,
|
|
"logps/chosen": -0.24509501457214355,
|
|
"logps/rejected": -0.3073538541793823,
|
|
"loss": 0.6780762076377869,
|
|
"loss/core": 0.6689341068267822,
|
|
"loss/preference_sft": 0.24509501457214355,
|
|
"loss/reference_anchor": 0.4325965940952301,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.8887115716934204,
|
|
"rewards/margins": -0.9674921035766602,
|
|
"rewards/rejected": 2.856203556060791,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15812969207763672,
|
|
"drift/rejected_abs": 0.15588991343975067,
|
|
"epoch": 0.10511855700889924,
|
|
"grad_norm": 235.0,
|
|
"ht_mnpo/logit": 0.002391496207565069,
|
|
"ht_mnpo/residual": -0.0051085082814097404,
|
|
"ht_mnpo/residual_abs": 0.14227387309074402,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.496607495886281e-07,
|
|
"logits/chosen": -2.202495574951172,
|
|
"logits/rejected": -1.9575474262237549,
|
|
"logps/chosen": -0.33234265446662903,
|
|
"logps/rejected": -0.3109672963619232,
|
|
"loss": 0.2439870834350586,
|
|
"loss/core": 0.239820197224617,
|
|
"loss/preference_sft": 0.33234265446662903,
|
|
"loss/reference_anchor": 0.1751091033220291,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.5793108940124512,
|
|
"rewards/margins": 0.02391493320465088,
|
|
"rewards/rejected": 1.5553959608078003,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25509947538375854,
|
|
"drift/rejected_abs": 0.1245686411857605,
|
|
"epoch": 0.10989667323657648,
|
|
"grad_norm": 1.7734375,
|
|
"ht_mnpo/logit": 0.13049247860908508,
|
|
"ht_mnpo/residual": 0.12299247831106186,
|
|
"ht_mnpo/residual_abs": 0.15432575345039368,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4945884883122553e-07,
|
|
"logits/chosen": -2.340104341506958,
|
|
"logits/rejected": -2.0771660804748535,
|
|
"logps/chosen": -0.2742314338684082,
|
|
"logps/rejected": -0.2798416316509247,
|
|
"loss": 0.39075616002082825,
|
|
"loss/core": 0.3828144669532776,
|
|
"loss/preference_sft": 0.2742314338684082,
|
|
"loss/reference_anchor": 0.36966127157211304,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.5494778156280518,
|
|
"rewards/margins": 1.304924726486206,
|
|
"rewards/rejected": 1.2445532083511353,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19862906634807587,
|
|
"drift/rejected_abs": 0.09067542850971222,
|
|
"epoch": 0.11467478946425372,
|
|
"grad_norm": 588.0,
|
|
"ht_mnpo/logit": 0.1080908328294754,
|
|
"ht_mnpo/residual": 0.10059082508087158,
|
|
"ht_mnpo/residual_abs": 0.11183144152164459,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.492101441907714e-07,
|
|
"logits/chosen": -2.4141907691955566,
|
|
"logits/rejected": -2.2686760425567627,
|
|
"logps/chosen": -0.2924870550632477,
|
|
"logps/rejected": -0.3066648840904236,
|
|
"loss": 0.27137768268585205,
|
|
"loss/core": 0.2637272775173187,
|
|
"loss/preference_sft": 0.2924870550632477,
|
|
"loss/reference_anchor": 0.35327258706092834,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.985629677772522,
|
|
"rewards/margins": 1.0809080600738525,
|
|
"rewards/rejected": 0.9047213792800903,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17624413967132568,
|
|
"drift/rejected_abs": 0.1153462678194046,
|
|
"epoch": 0.11945290569193096,
|
|
"grad_norm": 13.9375,
|
|
"ht_mnpo/logit": 0.06103396415710449,
|
|
"ht_mnpo/residual": 0.053533971309661865,
|
|
"ht_mnpo/residual_abs": 0.07010561972856522,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4891472919490977e-07,
|
|
"logits/chosen": -2.2220981121063232,
|
|
"logits/rejected": -2.035374879837036,
|
|
"logps/chosen": -0.3033459782600403,
|
|
"logps/rejected": -0.3098070025444031,
|
|
"loss": 0.041915275156497955,
|
|
"loss/core": 0.038992445915937424,
|
|
"loss/preference_sft": 0.3033459782600403,
|
|
"loss/reference_anchor": 0.11580680310726166,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.7615543603897095,
|
|
"rewards/margins": 0.6103397011756897,
|
|
"rewards/rejected": 1.151214361190796,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2372765839099884,
|
|
"drift/rejected_abs": 0.15658681094646454,
|
|
"epoch": 0.12423102191960819,
|
|
"grad_norm": 2.78125,
|
|
"ht_mnpo/logit": 0.0809415802359581,
|
|
"ht_mnpo/residual": 0.07344158738851547,
|
|
"ht_mnpo/residual_abs": 0.09841513633728027,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4857271493713894e-07,
|
|
"logits/chosen": -2.1383440494537354,
|
|
"logits/rejected": -1.9111261367797852,
|
|
"logps/chosen": -0.2732178568840027,
|
|
"logps/rejected": -0.29032042622566223,
|
|
"loss": 0.0999687910079956,
|
|
"loss/core": 0.09330854564905167,
|
|
"loss/preference_sft": 0.2732178568840027,
|
|
"loss/reference_anchor": 0.30569005012512207,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3718535900115967,
|
|
"rewards/margins": 0.8094158172607422,
|
|
"rewards/rejected": 1.5624377727508545,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15119370818138123,
|
|
"drift/rejected_abs": 0.09558363258838654,
|
|
"epoch": 0.12900913814728543,
|
|
"grad_norm": 15.125,
|
|
"ht_mnpo/logit": 0.05558431148529053,
|
|
"ht_mnpo/residual": 0.048084307461977005,
|
|
"ht_mnpo/residual_abs": 0.0851782038807869,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.481842300350339e-07,
|
|
"logits/chosen": -2.00144624710083,
|
|
"logits/rejected": -1.7436069250106812,
|
|
"logps/chosen": -0.26819977164268494,
|
|
"logps/rejected": -0.2890457510948181,
|
|
"loss": 0.04468156024813652,
|
|
"loss/core": 0.04284118860960007,
|
|
"loss/preference_sft": 0.26819977164268494,
|
|
"loss/reference_anchor": 0.06519876420497894,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.51114821434021,
|
|
"rewards/margins": 0.5558429956436157,
|
|
"rewards/rejected": 0.9553052186965942,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14579074084758759,
|
|
"drift/rejected_abs": 0.10920121520757675,
|
|
"epoch": 0.13378725437496267,
|
|
"grad_norm": 0.294921875,
|
|
"ht_mnpo/logit": 0.03659818693995476,
|
|
"ht_mnpo/residual": 0.029098186641931534,
|
|
"ht_mnpo/residual_abs": 0.04317057505249977,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4774942058187854e-07,
|
|
"logits/chosen": -2.337130308151245,
|
|
"logits/rejected": -2.108213424682617,
|
|
"logps/chosen": -0.296450674533844,
|
|
"logps/rejected": -0.2900570333003998,
|
|
"loss": 0.009442105889320374,
|
|
"loss/core": 0.00706028938293457,
|
|
"loss/preference_sft": 0.296450674533844,
|
|
"loss/reference_anchor": 0.08944575488567352,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.4567911624908447,
|
|
"rewards/margins": 0.3659818470478058,
|
|
"rewards/rejected": 1.0908094644546509,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19297416508197784,
|
|
"drift/rejected_abs": 0.15170451998710632,
|
|
"epoch": 0.1385653706026399,
|
|
"grad_norm": 5.90625,
|
|
"ht_mnpo/logit": 0.041436947882175446,
|
|
"ht_mnpo/residual": 0.03393695130944252,
|
|
"ht_mnpo/residual_abs": 0.0991707444190979,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.472684500917257e-07,
|
|
"logits/chosen": -2.286118984222412,
|
|
"logits/rejected": -2.0098800659179688,
|
|
"logps/chosen": -0.2768682837486267,
|
|
"logps/rejected": -0.2945496141910553,
|
|
"loss": 0.08796399086713791,
|
|
"loss/core": 0.08381696045398712,
|
|
"loss/preference_sft": 0.2768682837486267,
|
|
"loss/reference_anchor": 0.1796645224094391,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.9294698238372803,
|
|
"rewards/margins": 0.4143694341182709,
|
|
"rewards/rejected": 1.5151002407073975,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1415017545223236,
|
|
"drift/rejected_abs": 0.08456171303987503,
|
|
"epoch": 0.14334348683031714,
|
|
"grad_norm": 1.90625,
|
|
"ht_mnpo/logit": 0.05727854371070862,
|
|
"ht_mnpo/residual": 0.04977855831384659,
|
|
"ht_mnpo/residual_abs": 0.05690733715891838,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.467414994379065e-07,
|
|
"logits/chosen": -2.3385872840881348,
|
|
"logits/rejected": -2.1087775230407715,
|
|
"logps/chosen": -0.2883468270301819,
|
|
"logps/rejected": -0.29340860247612,
|
|
"loss": 0.02342948503792286,
|
|
"loss/core": 0.021441509947180748,
|
|
"loss/preference_sft": 0.2883468270301819,
|
|
"loss/reference_anchor": 0.07056401669979095,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4143787622451782,
|
|
"rewards/margins": 0.572785496711731,
|
|
"rewards/rejected": 0.8415932655334473,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19519026577472687,
|
|
"drift/rejected_abs": 0.1824832707643509,
|
|
"epoch": 0.14812160305799438,
|
|
"grad_norm": 27.25,
|
|
"ht_mnpo/logit": 0.01290883868932724,
|
|
"ht_mnpo/residual": 0.005408839322626591,
|
|
"ht_mnpo/residual_abs": 0.10687340795993805,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4616876678501114e-07,
|
|
"logits/chosen": -2.172908306121826,
|
|
"logits/rejected": -1.944345235824585,
|
|
"logps/chosen": -0.30824723839759827,
|
|
"logps/rejected": -0.32008838653564453,
|
|
"loss": 0.06261222064495087,
|
|
"loss/core": 0.05846996232867241,
|
|
"loss/preference_sft": 0.30824723839759827,
|
|
"loss/reference_anchor": 0.17628782987594604,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.950910210609436,
|
|
"rewards/margins": 0.12908843159675598,
|
|
"rewards/rejected": 1.8218215703964233,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.177214115858078,
|
|
"drift/rejected_abs": 0.12131655216217041,
|
|
"epoch": 0.15289971928567161,
|
|
"grad_norm": 6.65625,
|
|
"ht_mnpo/logit": 0.05621614307165146,
|
|
"ht_mnpo/residual": 0.04871614649891853,
|
|
"ht_mnpo/residual_abs": 0.07616520673036575,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4555046751436735e-07,
|
|
"logits/chosen": -2.0418126583099365,
|
|
"logits/rejected": -1.8054478168487549,
|
|
"logps/chosen": -0.2701527178287506,
|
|
"logps/rejected": -0.2704584002494812,
|
|
"loss": 0.036772746592760086,
|
|
"loss/core": 0.034319765865802765,
|
|
"loss/preference_sft": 0.2701527178287506,
|
|
"loss/reference_anchor": 0.09563368558883667,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7721397876739502,
|
|
"rewards/margins": 0.5621614456176758,
|
|
"rewards/rejected": 1.2099783420562744,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23195543885231018,
|
|
"drift/rejected_abs": 0.14278283715248108,
|
|
"epoch": 0.15767783551334885,
|
|
"grad_norm": 3.5,
|
|
"ht_mnpo/logit": 0.089602030813694,
|
|
"ht_mnpo/residual": 0.08210203796625137,
|
|
"ht_mnpo/residual_abs": 0.15943965315818787,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4488683414304425e-07,
|
|
"logits/chosen": -2.2963273525238037,
|
|
"logits/rejected": -2.1011674404144287,
|
|
"logps/chosen": -0.2527472972869873,
|
|
"logps/rejected": -0.2691109776496887,
|
|
"loss": 0.32412639260292053,
|
|
"loss/core": 0.318270742893219,
|
|
"loss/preference_sft": 0.2527472972869873,
|
|
"loss/reference_anchor": 0.26750943064689636,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.3194217681884766,
|
|
"rewards/margins": 0.8960202932357788,
|
|
"rewards/rejected": 1.4234017133712769,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23300659656524658,
|
|
"drift/rejected_abs": 0.22859875857830048,
|
|
"epoch": 0.1624559517410261,
|
|
"grad_norm": 1.578125,
|
|
"ht_mnpo/logit": 0.004623569548130035,
|
|
"ht_mnpo/residual": -0.0028764219023287296,
|
|
"ht_mnpo/residual_abs": 0.1669018566608429,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4417811623641203e-07,
|
|
"logits/chosen": -2.144458055496216,
|
|
"logits/rejected": -1.9285306930541992,
|
|
"logps/chosen": -0.27059751749038696,
|
|
"logps/rejected": -0.2944408059120178,
|
|
"loss": 0.4071100354194641,
|
|
"loss/core": 0.397773414850235,
|
|
"loss/preference_sft": 0.27059751749038696,
|
|
"loss/reference_anchor": 0.4397706985473633,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.330066204071045,
|
|
"rewards/margins": 0.0462358221411705,
|
|
"rewards/rejected": 2.283830165863037,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23780396580696106,
|
|
"drift/rejected_abs": 0.21040399372577667,
|
|
"epoch": 0.16723406796870333,
|
|
"grad_norm": 0.765625,
|
|
"ht_mnpo/logit": 0.027342364192008972,
|
|
"ht_mnpo/residual": 0.019842345267534256,
|
|
"ht_mnpo/residual_abs": 0.26941126585006714,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4342458031429113e-07,
|
|
"logits/chosen": -2.155928134918213,
|
|
"logits/rejected": -1.8418967723846436,
|
|
"logps/chosen": -0.28269365429878235,
|
|
"logps/rejected": -0.3167445659637451,
|
|
"loss": 0.8618923425674438,
|
|
"loss/core": 0.8492777943611145,
|
|
"loss/preference_sft": 0.28269365429878235,
|
|
"loss/reference_anchor": 0.6024563908576965,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3754687309265137,
|
|
"rewards/margins": 0.273423433303833,
|
|
"rewards/rejected": 2.1020452976226807,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17630836367607117,
|
|
"drift/rejected_abs": 0.15115013718605042,
|
|
"epoch": 0.17201218419638056,
|
|
"grad_norm": 3.4375,
|
|
"ht_mnpo/logit": 0.02521784044802189,
|
|
"ht_mnpo/residual": 0.017717847600579262,
|
|
"ht_mnpo/residual_abs": 0.11181072145700455,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4262650975072444e-07,
|
|
"logits/chosen": -2.3172078132629395,
|
|
"logits/rejected": -2.0724923610687256,
|
|
"logps/chosen": -0.2543174624443054,
|
|
"logps/rejected": -0.24170401692390442,
|
|
"loss": 0.10301873832941055,
|
|
"loss/core": 0.09923148900270462,
|
|
"loss/preference_sft": 0.2543174624443054,
|
|
"loss/reference_anchor": 0.16393068432807922,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.7623436450958252,
|
|
"rewards/margins": 0.252178430557251,
|
|
"rewards/rejected": 1.5101648569107056,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4186461567878723,
|
|
"drift/rejected_abs": 0.10205858945846558,
|
|
"epoch": 0.17679030042405783,
|
|
"grad_norm": 32.0,
|
|
"ht_mnpo/logit": 0.32172200083732605,
|
|
"ht_mnpo/residual": 0.3142220079898834,
|
|
"ht_mnpo/residual_abs": 0.3194965720176697,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.417842046674122e-07,
|
|
"logits/chosen": -2.145289421081543,
|
|
"logits/rejected": -1.9773670434951782,
|
|
"logps/chosen": -0.3100564777851105,
|
|
"logps/rejected": -0.31372490525245667,
|
|
"loss": 1.5905460119247437,
|
|
"loss/core": 1.5711582899093628,
|
|
"loss/preference_sft": 0.3100564777851105,
|
|
"loss/reference_anchor": 0.9383944272994995,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.18438720703125,
|
|
"rewards/margins": 3.217219829559326,
|
|
"rewards/rejected": 0.9671671986579895,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14292044937610626,
|
|
"drift/rejected_abs": 0.08453439176082611,
|
|
"epoch": 0.18156841665173507,
|
|
"grad_norm": 6.78125,
|
|
"ht_mnpo/logit": 0.05871760845184326,
|
|
"ht_mnpo/residual": 0.051217615604400635,
|
|
"ht_mnpo/residual_abs": 0.06432562321424484,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4089798182084843e-07,
|
|
"logits/chosen": -2.21405291557312,
|
|
"logits/rejected": -2.0281381607055664,
|
|
"logps/chosen": -0.2850034534931183,
|
|
"logps/rejected": -0.29590287804603577,
|
|
"loss": 0.020889047533273697,
|
|
"loss/core": 0.01923513039946556,
|
|
"loss/preference_sft": 0.2850034534931183,
|
|
"loss/reference_anchor": 0.054195452481508255,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4286693334579468,
|
|
"rewards/margins": 0.5871760249137878,
|
|
"rewards/rejected": 0.8414933085441589,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2953607738018036,
|
|
"drift/rejected_abs": 0.0970875471830368,
|
|
"epoch": 0.1863465328794123,
|
|
"grad_norm": 0.77734375,
|
|
"ht_mnpo/logit": 0.2047409564256668,
|
|
"ht_mnpo/residual": 0.19724097847938538,
|
|
"ht_mnpo/residual_abs": 0.2064896821975708,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3996817448320195e-07,
|
|
"logits/chosen": -2.3270435333251953,
|
|
"logits/rejected": -2.1133506298065186,
|
|
"logps/chosen": -0.315519243478775,
|
|
"logps/rejected": -0.26455676555633545,
|
|
"loss": 0.8637374639511108,
|
|
"loss/core": 0.8507298231124878,
|
|
"loss/preference_sft": 0.315519243478775,
|
|
"loss/reference_anchor": 0.6188250780105591,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9532313346862793,
|
|
"rewards/margins": 2.047409772872925,
|
|
"rewards/rejected": 0.905821681022644,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1497114598751068,
|
|
"drift/rejected_abs": 0.10208232700824738,
|
|
"epoch": 0.19112464910708954,
|
|
"grad_norm": 2.9375,
|
|
"ht_mnpo/logit": 0.04762087017297745,
|
|
"ht_mnpo/residual": 0.04012087732553482,
|
|
"ht_mnpo/residual_abs": 0.061163824051618576,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3899513231698607e-07,
|
|
"logits/chosen": -2.379368305206299,
|
|
"logits/rejected": -2.0901029109954834,
|
|
"logps/chosen": -0.2885080873966217,
|
|
"logps/rejected": -0.29427370429039,
|
|
"loss": 0.01607438549399376,
|
|
"loss/core": 0.014033469371497631,
|
|
"loss/preference_sft": 0.2885080873966217,
|
|
"loss/reference_anchor": 0.07319486886262894,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.4958560466766357,
|
|
"rewards/margins": 0.4762086868286133,
|
|
"rewards/rejected": 1.0196473598480225,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22366979718208313,
|
|
"drift/rejected_abs": 0.16261938214302063,
|
|
"epoch": 0.19590276533476678,
|
|
"grad_norm": 7.59375,
|
|
"ht_mnpo/logit": 0.06350360065698624,
|
|
"ht_mnpo/residual": 0.05600360780954361,
|
|
"ht_mnpo/residual_abs": 0.15254007279872894,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3797922124356506e-07,
|
|
"logits/chosen": -2.0374624729156494,
|
|
"logits/rejected": -1.8821849822998047,
|
|
"logps/chosen": -0.30214640498161316,
|
|
"logps/rejected": -0.32250362634658813,
|
|
"loss": 0.23545308411121368,
|
|
"loss/core": 0.2295982837677002,
|
|
"loss/preference_sft": 0.30214640498161316,
|
|
"loss/reference_anchor": 0.26252511143684387,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2357821464538574,
|
|
"rewards/margins": 0.6350361108779907,
|
|
"rewards/rejected": 1.6007457971572876,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16679704189300537,
|
|
"drift/rejected_abs": 0.1574680507183075,
|
|
"epoch": 0.200680881562444,
|
|
"grad_norm": 13.8125,
|
|
"ht_mnpo/logit": 0.009180588647723198,
|
|
"ht_mnpo/residual": 0.001680587651208043,
|
|
"ht_mnpo/residual_abs": 0.08262274414300919,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3692082330554585e-07,
|
|
"logits/chosen": -2.282926321029663,
|
|
"logits/rejected": -2.038118362426758,
|
|
"logps/chosen": -0.3618142306804657,
|
|
"logps/rejected": -0.2868843674659729,
|
|
"loss": 0.07468167692422867,
|
|
"loss/core": 0.07061126083135605,
|
|
"loss/preference_sft": 0.3618142306804657,
|
|
"loss/reference_anchor": 0.1673388034105301,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 1.6660865545272827,
|
|
"rewards/margins": 0.09180586040019989,
|
|
"rewards/rejected": 1.5742807388305664,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2035273313522339,
|
|
"drift/rejected_abs": 0.13128599524497986,
|
|
"epoch": 0.20545899779012125,
|
|
"grad_norm": 8.875,
|
|
"ht_mnpo/logit": 0.07242141664028168,
|
|
"ht_mnpo/residual": 0.06492140889167786,
|
|
"ht_mnpo/residual_abs": 0.11575053632259369,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3582033652310765e-07,
|
|
"logits/chosen": -2.10392689704895,
|
|
"logits/rejected": -1.9740612506866455,
|
|
"logps/chosen": -0.27785545587539673,
|
|
"logps/rejected": -0.2743791937828064,
|
|
"loss": 0.07868362963199615,
|
|
"loss/core": 0.07500644028186798,
|
|
"loss/preference_sft": 0.27785545587539673,
|
|
"loss/reference_anchor": 0.15607400238513947,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0350780487060547,
|
|
"rewards/margins": 0.7242141962051392,
|
|
"rewards/rejected": 1.3108640909194946,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27838942408561707,
|
|
"drift/rejected_abs": 0.20967824757099152,
|
|
"epoch": 0.2102371140177985,
|
|
"grad_norm": 3.1875,
|
|
"ht_mnpo/logit": 0.06874275207519531,
|
|
"ht_mnpo/residual": 0.06124274805188179,
|
|
"ht_mnpo/residual_abs": 0.17890474200248718,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.346781747443227e-07,
|
|
"logits/chosen": -2.0583040714263916,
|
|
"logits/rejected": -1.754565954208374,
|
|
"logps/chosen": -0.29740554094314575,
|
|
"logps/rejected": -0.28887122869491577,
|
|
"loss": 0.35021087527275085,
|
|
"loss/core": 0.339802622795105,
|
|
"loss/preference_sft": 0.29740554094314575,
|
|
"loss/reference_anchor": 0.490673303604126,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": 2.7822656631469727,
|
|
"rewards/margins": 0.6874275803565979,
|
|
"rewards/rejected": 2.0948376655578613,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20137110352516174,
|
|
"drift/rejected_abs": 0.14107075333595276,
|
|
"epoch": 0.21501523024547572,
|
|
"grad_norm": 2.234375,
|
|
"ht_mnpo/logit": 0.0653047114610672,
|
|
"ht_mnpo/residual": 0.057804714888334274,
|
|
"ht_mnpo/residual_abs": 0.1389622986316681,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3349476748952508e-07,
|
|
"logits/chosen": -2.24853515625,
|
|
"logits/rejected": -2.0200893878936768,
|
|
"logps/chosen": -0.27711716294288635,
|
|
"logps/rejected": -0.29098328948020935,
|
|
"loss": 0.17267122864723206,
|
|
"loss/core": 0.1683366894721985,
|
|
"loss/preference_sft": 0.27711716294288635,
|
|
"loss/reference_anchor": 0.18901614844799042,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.0136349201202393,
|
|
"rewards/margins": 0.6530472040176392,
|
|
"rewards/rejected": 1.3605875968933105,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15755267441272736,
|
|
"drift/rejected_abs": 0.16204038262367249,
|
|
"epoch": 0.21979334647315296,
|
|
"grad_norm": 8.5625,
|
|
"ht_mnpo/logit": -0.004402917809784412,
|
|
"ht_mnpo/residual": -0.011902913451194763,
|
|
"ht_mnpo/residual_abs": 0.13799583911895752,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3227055978978545e-07,
|
|
"logits/chosen": -2.2092113494873047,
|
|
"logits/rejected": -1.9548333883285522,
|
|
"logps/chosen": -0.27587762475013733,
|
|
"logps/rejected": -0.2882309854030609,
|
|
"loss": 0.20742078125476837,
|
|
"loss/core": 0.20317880809307098,
|
|
"loss/preference_sft": 0.27587762475013733,
|
|
"loss/reference_anchor": 0.18451109528541565,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.575390338897705,
|
|
"rewards/margins": -0.04402913898229599,
|
|
"rewards/rejected": 1.6194193363189697,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22635149955749512,
|
|
"drift/rejected_abs": 0.1525922268629074,
|
|
"epoch": 0.2245714627008302,
|
|
"grad_norm": 478.0,
|
|
"ht_mnpo/logit": 0.07377196848392487,
|
|
"ht_mnpo/residual": 0.06627197563648224,
|
|
"ht_mnpo/residual_abs": 0.17167456448078156,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3100601201955321e-07,
|
|
"logits/chosen": -2.1003198623657227,
|
|
"logits/rejected": -1.934924840927124,
|
|
"logps/chosen": -0.2860856354236603,
|
|
"logps/rejected": -0.2972342371940613,
|
|
"loss": 0.60406893491745,
|
|
"loss/core": 0.5958213806152344,
|
|
"loss/preference_sft": 0.2860856354236603,
|
|
"loss/reference_anchor": 0.38376861810684204,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.2631850242614746,
|
|
"rewards/margins": 0.7377198338508606,
|
|
"rewards/rejected": 1.5254653692245483,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13833805918693542,
|
|
"drift/rejected_abs": 0.08581127971410751,
|
|
"epoch": 0.22934957892850744,
|
|
"grad_norm": 6.8125,
|
|
"ht_mnpo/logit": 0.05265297740697861,
|
|
"ht_mnpo/residual": 0.04515298455953598,
|
|
"ht_mnpo/residual_abs": 0.08135689049959183,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2970159972352864e-07,
|
|
"logits/chosen": -2.3458266258239746,
|
|
"logits/rejected": -2.1107969284057617,
|
|
"logps/chosen": -0.2840951085090637,
|
|
"logps/rejected": -0.27419576048851013,
|
|
"loss": 0.03435717895627022,
|
|
"loss/core": 0.03234320878982544,
|
|
"loss/preference_sft": 0.2840951085090637,
|
|
"loss/reference_anchor": 0.0722891315817833,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.3823360204696655,
|
|
"rewards/margins": 0.5265297889709473,
|
|
"rewards/rejected": 0.8558060526847839,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17416642606258392,
|
|
"drift/rejected_abs": 0.14134536683559418,
|
|
"epoch": 0.23412769515618467,
|
|
"grad_norm": 40.0,
|
|
"ht_mnpo/logit": 0.03294435888528824,
|
|
"ht_mnpo/residual": 0.025444358587265015,
|
|
"ht_mnpo/residual_abs": 0.11847629398107529,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2835781343782966e-07,
|
|
"logits/chosen": -2.307325839996338,
|
|
"logits/rejected": -2.0234155654907227,
|
|
"logps/chosen": -0.3149130046367645,
|
|
"logps/rejected": -0.30677077174186707,
|
|
"loss": 0.1159428134560585,
|
|
"loss/core": 0.11232848465442657,
|
|
"loss/preference_sft": 0.3149130046367645,
|
|
"loss/reference_anchor": 0.1492251455783844,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.7409662008285522,
|
|
"rewards/margins": 0.3294435739517212,
|
|
"rewards/rejected": 1.4115229845046997,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2660543918609619,
|
|
"drift/rejected_abs": 0.11109892278909683,
|
|
"epoch": 0.2389058113838619,
|
|
"grad_norm": 868.0,
|
|
"ht_mnpo/logit": 0.1549011766910553,
|
|
"ht_mnpo/residual": 0.14740118384361267,
|
|
"ht_mnpo/residual_abs": 0.15847142040729523,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2697515850552152e-07,
|
|
"logits/chosen": -2.3213107585906982,
|
|
"logits/rejected": -2.1739933490753174,
|
|
"logps/chosen": -0.25932732224464417,
|
|
"logps/rejected": -0.2790648937225342,
|
|
"loss": 0.598106324672699,
|
|
"loss/core": 0.5861698389053345,
|
|
"loss/preference_sft": 0.25932732224464417,
|
|
"loss/reference_anchor": 0.5708891153335571,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6590445041656494,
|
|
"rewards/margins": 1.5490118265151978,
|
|
"rewards/rejected": 1.1100324392318726,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1992466151714325,
|
|
"drift/rejected_abs": 0.11664634943008423,
|
|
"epoch": 0.24368392761153915,
|
|
"grad_norm": 4.4375,
|
|
"ht_mnpo/logit": 0.08275730162858963,
|
|
"ht_mnpo/residual": 0.0752573013305664,
|
|
"ht_mnpo/residual_abs": 0.08343817293643951,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2555415488657775e-07,
|
|
"logits/chosen": -2.1033692359924316,
|
|
"logits/rejected": -1.9849478006362915,
|
|
"logps/chosen": -0.24639995396137238,
|
|
"logps/rejected": -0.2411382645368576,
|
|
"loss": 0.04728394001722336,
|
|
"loss/core": 0.0447533018887043,
|
|
"loss/preference_sft": 0.24639995396137238,
|
|
"loss/reference_anchor": 0.10189200937747955,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.9907464981079102,
|
|
"rewards/margins": 0.8275730013847351,
|
|
"rewards/rejected": 1.1631735563278198,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29991379380226135,
|
|
"drift/rejected_abs": 0.18915988504886627,
|
|
"epoch": 0.24846204383921638,
|
|
"grad_norm": 25.25,
|
|
"ht_mnpo/logit": 0.11084876209497452,
|
|
"ht_mnpo/residual": 0.1033487543463707,
|
|
"ht_mnpo/residual_abs": 0.22364541888237,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.240953369623447e-07,
|
|
"logits/chosen": -2.131293535232544,
|
|
"logits/rejected": -1.8576065301895142,
|
|
"logps/chosen": -0.2812115550041199,
|
|
"logps/rejected": -0.30115509033203125,
|
|
"loss": 0.7878208160400391,
|
|
"loss/core": 0.7746709585189819,
|
|
"loss/preference_sft": 0.2812115550041199,
|
|
"loss/reference_anchor": 0.6293684840202332,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.9983861446380615,
|
|
"rewards/margins": 1.108487606048584,
|
|
"rewards/rejected": 1.8898985385894775,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13449910283088684,
|
|
"drift/rejected_abs": 0.12530843913555145,
|
|
"epoch": 0.25324016006689365,
|
|
"grad_norm": 0.462890625,
|
|
"ht_mnpo/logit": 0.009142006747424603,
|
|
"ht_mnpo/residual": 0.0016420104075223207,
|
|
"ht_mnpo/residual_abs": 0.09060578048229218,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.225992533345824e-07,
|
|
"logits/chosen": -2.140869379043579,
|
|
"logits/rejected": -1.6922798156738281,
|
|
"logps/chosen": -0.29281654953956604,
|
|
"logps/rejected": -0.2915118336677551,
|
|
"loss": 0.042861003428697586,
|
|
"loss/core": 0.04084916412830353,
|
|
"loss/preference_sft": 0.29281654953956604,
|
|
"loss/reference_anchor": 0.07131018489599228,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 1.3426874876022339,
|
|
"rewards/margins": 0.09142010658979416,
|
|
"rewards/rejected": 1.2512675523757935,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20011694729328156,
|
|
"drift/rejected_abs": 0.09070627391338348,
|
|
"epoch": 0.25801827629457086,
|
|
"grad_norm": 11.375,
|
|
"ht_mnpo/logit": 0.11147568374872208,
|
|
"ht_mnpo/residual": 0.10397566854953766,
|
|
"ht_mnpo/residual_abs": 0.1260581612586975,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.210664666191579e-07,
|
|
"logits/chosen": -2.4403769969940186,
|
|
"logits/rejected": -2.233523368835449,
|
|
"logps/chosen": -0.2794303297996521,
|
|
"logps/rejected": -0.28702133893966675,
|
|
"loss": 0.2465023547410965,
|
|
"loss/core": 0.24205832183361053,
|
|
"loss/preference_sft": 0.2794303297996521,
|
|
"loss/reference_anchor": 0.19425857067108154,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9994302988052368,
|
|
"rewards/margins": 1.1147568225860596,
|
|
"rewards/rejected": 0.8846734166145325,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4535955488681793,
|
|
"drift/rejected_abs": 0.2866347134113312,
|
|
"epoch": 0.2627963925222481,
|
|
"grad_norm": 7.96875,
|
|
"ht_mnpo/logit": 0.16719956696033478,
|
|
"ht_mnpo/residual": 0.15969957411289215,
|
|
"ht_mnpo/residual_abs": 0.29034724831581116,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1949755323446848e-07,
|
|
"logits/chosen": -2.2299437522888184,
|
|
"logits/rejected": -1.9355762004852295,
|
|
"logps/chosen": -0.28128141164779663,
|
|
"logps/rejected": -0.28012967109680176,
|
|
"loss": 0.9513923525810242,
|
|
"loss/core": 0.9261605143547058,
|
|
"loss/preference_sft": 0.28128141164779663,
|
|
"loss/reference_anchor": 1.2334659099578857,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.535092353820801,
|
|
"rewards/margins": 1.671995759010315,
|
|
"rewards/rejected": 2.8630967140197754,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24227941036224365,
|
|
"drift/rejected_abs": 0.1327737718820572,
|
|
"epoch": 0.26757450874992533,
|
|
"grad_norm": 8.125,
|
|
"ht_mnpo/logit": 0.10931800305843353,
|
|
"ht_mnpo/residual": 0.1018180102109909,
|
|
"ht_mnpo/residual_abs": 0.11589553207159042,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1789310318467426e-07,
|
|
"logits/chosen": -2.0422604084014893,
|
|
"logits/rejected": -1.9183323383331299,
|
|
"logps/chosen": -0.2838364541530609,
|
|
"logps/rejected": -0.27868276834487915,
|
|
"loss": 0.17103958129882812,
|
|
"loss/core": 0.16334925591945648,
|
|
"loss/preference_sft": 0.2838364541530609,
|
|
"loss/reference_anchor": 0.356132447719574,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.420574903488159,
|
|
"rewards/margins": 1.0931799411773682,
|
|
"rewards/rejected": 1.327394962310791,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2416619062423706,
|
|
"drift/rejected_abs": 0.1695837825536728,
|
|
"epoch": 0.2723526249776026,
|
|
"grad_norm": 5.875,
|
|
"ht_mnpo/logit": 0.07214997708797455,
|
|
"ht_mnpo/residual": 0.06464996933937073,
|
|
"ht_mnpo/residual_abs": 0.09243568778038025,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1625371983782182e-07,
|
|
"logits/chosen": -2.091132164001465,
|
|
"logits/rejected": -1.8592939376831055,
|
|
"logps/chosen": -0.3188819885253906,
|
|
"logps/rejected": -0.3187660574913025,
|
|
"loss": 0.0901908352971077,
|
|
"loss/core": 0.07827536761760712,
|
|
"loss/preference_sft": 0.3188819885253906,
|
|
"loss/reference_anchor": 0.5638847947120667,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.415431499481201,
|
|
"rewards/margins": 0.7214997410774231,
|
|
"rewards/rejected": 1.6939315795898438,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20275557041168213,
|
|
"drift/rejected_abs": 0.165045827627182,
|
|
"epoch": 0.2771307412052798,
|
|
"grad_norm": 18.125,
|
|
"ht_mnpo/logit": 0.037836022675037384,
|
|
"ht_mnpo/residual": 0.030336027964949608,
|
|
"ht_mnpo/residual_abs": 0.08644749969244003,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.14580019698942e-07,
|
|
"logits/chosen": -2.215244770050049,
|
|
"logits/rejected": -1.9540691375732422,
|
|
"logps/chosen": -0.28964218497276306,
|
|
"logps/rejected": -0.28602391481399536,
|
|
"loss": 0.04661395773291588,
|
|
"loss/core": 0.04280799999833107,
|
|
"loss/preference_sft": 0.28964218497276306,
|
|
"loss/reference_anchor": 0.161333829164505,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.027524471282959,
|
|
"rewards/margins": 0.3783602714538574,
|
|
"rewards/rejected": 1.649163842201233,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2054920196533203,
|
|
"drift/rejected_abs": 0.2024458944797516,
|
|
"epoch": 0.28190885743295707,
|
|
"grad_norm": 448.0,
|
|
"ht_mnpo/logit": 0.0030287965200841427,
|
|
"ht_mnpo/residual": -0.004471203777939081,
|
|
"ht_mnpo/residual_abs": 0.1715690791606903,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1287263217820773e-07,
|
|
"logits/chosen": -2.1912224292755127,
|
|
"logits/rejected": -2.0103259086608887,
|
|
"logps/chosen": -0.27507418394088745,
|
|
"logps/rejected": -0.2898535132408142,
|
|
"loss": 0.5342413187026978,
|
|
"loss/core": 0.5257778763771057,
|
|
"loss/preference_sft": 0.27507418394088745,
|
|
"loss/reference_anchor": 0.39566391706466675,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.054013252258301,
|
|
"rewards/margins": 0.030287981033325195,
|
|
"rewards/rejected": 2.0237250328063965,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32648366689682007,
|
|
"drift/rejected_abs": 0.14543873071670532,
|
|
"epoch": 0.2866869736606343,
|
|
"grad_norm": 7.25,
|
|
"ht_mnpo/logit": 0.19056273996829987,
|
|
"ht_mnpo/residual": 0.18306274712085724,
|
|
"ht_mnpo/residual_abs": 0.22434112429618835,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.111321993542385e-07,
|
|
"logits/chosen": -2.2440693378448486,
|
|
"logits/rejected": -1.9663177728652954,
|
|
"logps/chosen": -0.29654619097709656,
|
|
"logps/rejected": -0.30812662839889526,
|
|
"loss": 1.0591758489608765,
|
|
"loss/core": 1.0461701154708862,
|
|
"loss/preference_sft": 0.29654619097709656,
|
|
"loss/reference_anchor": 0.6206279397010803,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.2646141052246094,
|
|
"rewards/margins": 1.9056276082992554,
|
|
"rewards/rejected": 1.3589863777160645,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3030497133731842,
|
|
"drift/rejected_abs": 0.14767694473266602,
|
|
"epoch": 0.29146508988831155,
|
|
"grad_norm": 1984.0,
|
|
"ht_mnpo/logit": 0.15544824302196503,
|
|
"ht_mnpo/residual": 0.1479482352733612,
|
|
"ht_mnpo/residual_abs": 0.16500267386436462,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0935937573264096e-07,
|
|
"logits/chosen": -2.2528464794158936,
|
|
"logits/rejected": -2.066157579421997,
|
|
"logps/chosen": -0.2954830527305603,
|
|
"logps/rejected": -0.2760084271430969,
|
|
"loss": 0.5691508650779724,
|
|
"loss/core": 0.5540502071380615,
|
|
"loss/preference_sft": 0.2954830527305603,
|
|
"loss/reference_anchor": 0.7254848480224609,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.027735471725464,
|
|
"rewards/margins": 1.5544825792312622,
|
|
"rewards/rejected": 1.4732530117034912,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24608822166919708,
|
|
"drift/rejected_abs": 0.19582870602607727,
|
|
"epoch": 0.29624320611598876,
|
|
"grad_norm": 18.0,
|
|
"ht_mnpo/logit": 0.050264328718185425,
|
|
"ht_mnpo/residual": 0.0427643321454525,
|
|
"ht_mnpo/residual_abs": 0.16030198335647583,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0755482799987596e-07,
|
|
"logits/chosen": -2.362098455429077,
|
|
"logits/rejected": -2.1859664916992188,
|
|
"logps/chosen": -0.27773889899253845,
|
|
"logps/rejected": -0.2750205099582672,
|
|
"loss": 0.27516818046569824,
|
|
"loss/core": 0.26793110370635986,
|
|
"loss/preference_sft": 0.27773889899253845,
|
|
"loss/reference_anchor": 0.33407989144325256,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.458608388900757,
|
|
"rewards/margins": 0.5026432275772095,
|
|
"rewards/rejected": 1.9559653997421265,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16757844388484955,
|
|
"drift/rejected_abs": 0.13082993030548096,
|
|
"epoch": 0.301021322343666,
|
|
"grad_norm": 9.875,
|
|
"ht_mnpo/logit": 0.045271605253219604,
|
|
"ht_mnpo/residual": 0.03777160868048668,
|
|
"ht_mnpo/residual_abs": 0.13339407742023468,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0571923477254526e-07,
|
|
"logits/chosen": -2.4076950550079346,
|
|
"logits/rejected": -2.056607246398926,
|
|
"logps/chosen": -0.28617167472839355,
|
|
"logps/rejected": -0.2988688349723816,
|
|
"loss": 0.23253202438354492,
|
|
"loss/core": 0.22873583436012268,
|
|
"loss/preference_sft": 0.28617167472839355,
|
|
"loss/reference_anchor": 0.16119354963302612,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.6741405725479126,
|
|
"rewards/margins": 0.4527161717414856,
|
|
"rewards/rejected": 1.2214245796203613,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20812535285949707,
|
|
"drift/rejected_abs": 0.12873616814613342,
|
|
"epoch": 0.30579943857134323,
|
|
"grad_norm": 99.0,
|
|
"ht_mnpo/logit": 0.08008380234241486,
|
|
"ht_mnpo/residual": 0.07258381694555283,
|
|
"ht_mnpo/residual_abs": 0.16876870393753052,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.038532863421914e-07,
|
|
"logits/chosen": -2.2163033485412598,
|
|
"logits/rejected": -1.90713369846344,
|
|
"logps/chosen": -0.2852802872657776,
|
|
"logps/rejected": -0.29477205872535706,
|
|
"loss": 0.333568811416626,
|
|
"loss/core": 0.3274485170841217,
|
|
"loss/preference_sft": 0.2852802872657776,
|
|
"loss/reference_anchor": 0.277485728263855,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0812535285949707,
|
|
"rewards/margins": 0.8008379936218262,
|
|
"rewards/rejected": 1.2804152965545654,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24407429993152618,
|
|
"drift/rejected_abs": 0.11723603308200836,
|
|
"epoch": 0.3105775547990205,
|
|
"grad_norm": 438.0,
|
|
"ht_mnpo/logit": 0.13229139149188995,
|
|
"ht_mnpo/residual": 0.12479136139154434,
|
|
"ht_mnpo/residual_abs": 0.16577567160129547,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0195768441570726e-07,
|
|
"logits/chosen": -2.1131129264831543,
|
|
"logits/rejected": -1.9461160898208618,
|
|
"logps/chosen": -0.3240113854408264,
|
|
"logps/rejected": -0.3457443118095398,
|
|
"loss": 0.32235246896743774,
|
|
"loss/core": 0.3146476149559021,
|
|
"loss/preference_sft": 0.3240113854408264,
|
|
"loss/reference_anchor": 0.3528444468975067,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4372329711914062,
|
|
"rewards/margins": 1.3229138851165771,
|
|
"rewards/rejected": 1.114319086074829,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1336461305618286,
|
|
"drift/rejected_abs": 0.14605210721492767,
|
|
"epoch": 0.3153556710266977,
|
|
"grad_norm": 9.625,
|
|
"ht_mnpo/logit": -0.012384334579110146,
|
|
"ht_mnpo/residual": -0.01988433301448822,
|
|
"ht_mnpo/residual_abs": 0.10770714282989502,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0003314185145307e-07,
|
|
"logits/chosen": -2.1785032749176025,
|
|
"logits/rejected": -1.9495980739593506,
|
|
"logps/chosen": -0.27921396493911743,
|
|
"logps/rejected": -0.3031226396560669,
|
|
"loss": 0.2651267647743225,
|
|
"loss/core": 0.2610391080379486,
|
|
"loss/preference_sft": 0.27921396493911743,
|
|
"loss/reference_anchor": 0.1764618158340454,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.3336832523345947,
|
|
"rewards/margins": -0.12384340912103653,
|
|
"rewards/rejected": 1.457526445388794,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23569095134735107,
|
|
"drift/rejected_abs": 0.17359793186187744,
|
|
"epoch": 0.32013378725437497,
|
|
"grad_norm": 292.0,
|
|
"ht_mnpo/logit": 0.06208749860525131,
|
|
"ht_mnpo/residual": 0.05458749458193779,
|
|
"ht_mnpo/residual_abs": 0.13036298751831055,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.9808038239117913e-07,
|
|
"logits/chosen": -2.226792097091675,
|
|
"logits/rejected": -1.9871718883514404,
|
|
"logps/chosen": -0.2731962502002716,
|
|
"logps/rejected": -0.26592594385147095,
|
|
"loss": 0.16251419484615326,
|
|
"loss/core": 0.15205292403697968,
|
|
"loss/preference_sft": 0.2731962502002716,
|
|
"loss/reference_anchor": 0.49574416875839233,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.35514497756958,
|
|
"rewards/margins": 0.6208750009536743,
|
|
"rewards/rejected": 1.7342700958251953,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22859697043895721,
|
|
"drift/rejected_abs": 0.14668402075767517,
|
|
"epoch": 0.3249119034820522,
|
|
"grad_norm": 31.625,
|
|
"ht_mnpo/logit": 0.0824618712067604,
|
|
"ht_mnpo/residual": 0.07496186345815659,
|
|
"ht_mnpo/residual_abs": 0.17400358617305756,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.9610014038785646e-07,
|
|
"logits/chosen": -2.1349992752075195,
|
|
"logits/rejected": -2.008659839630127,
|
|
"logps/chosen": -0.29214566946029663,
|
|
"logps/rejected": -0.2824420630931854,
|
|
"loss": 0.28576305508613586,
|
|
"loss/core": 0.280608206987381,
|
|
"loss/preference_sft": 0.29214566946029663,
|
|
"loss/reference_anchor": 0.22852830588817596,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2851531505584717,
|
|
"rewards/margins": 0.8246185183525085,
|
|
"rewards/rejected": 1.4605344533920288,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25084584951400757,
|
|
"drift/rejected_abs": 0.26426196098327637,
|
|
"epoch": 0.32969001970972944,
|
|
"grad_norm": 1.6875,
|
|
"ht_mnpo/logit": -0.013153704814612865,
|
|
"ht_mnpo/residual": -0.020653702318668365,
|
|
"ht_mnpo/residual_abs": 0.19523727893829346,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.9409316052951657e-07,
|
|
"logits/chosen": -2.104379415512085,
|
|
"logits/rejected": -1.8748838901519775,
|
|
"logps/chosen": -0.29315048456192017,
|
|
"logps/rejected": -0.3299732506275177,
|
|
"loss": 0.4619339406490326,
|
|
"loss/core": 0.4512755274772644,
|
|
"loss/preference_sft": 0.29315048456192017,
|
|
"loss/reference_anchor": 0.503603458404541,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.508052349090576,
|
|
"rewards/margins": -0.1315370798110962,
|
|
"rewards/rejected": 2.639589309692383,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21669964492321014,
|
|
"drift/rejected_abs": 0.08468327671289444,
|
|
"epoch": 0.33446813593740665,
|
|
"grad_norm": 10.4375,
|
|
"ht_mnpo/logit": 0.13242420554161072,
|
|
"ht_mnpo/residual": 0.12492422014474869,
|
|
"ht_mnpo/residual_abs": 0.1702018678188324,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.920601975592047e-07,
|
|
"logits/chosen": -2.063258409500122,
|
|
"logits/rejected": -1.8462562561035156,
|
|
"logps/chosen": -0.30922695994377136,
|
|
"logps/rejected": -0.3094988763332367,
|
|
"loss": 0.6405929327011108,
|
|
"loss/core": 0.6327069997787476,
|
|
"loss/preference_sft": 0.30922695994377136,
|
|
"loss/reference_anchor": 0.3633735775947571,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.165689706802368,
|
|
"rewards/margins": 1.3242422342300415,
|
|
"rewards/rejected": 0.8414475321769714,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34972286224365234,
|
|
"drift/rejected_abs": 0.2043858766555786,
|
|
"epoch": 0.3392462521650839,
|
|
"grad_norm": 71.5,
|
|
"ht_mnpo/logit": 0.14536131918430328,
|
|
"ht_mnpo/residual": 0.13786129653453827,
|
|
"ht_mnpo/residual_abs": 0.18553060293197632,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.900020159911519e-07,
|
|
"logits/chosen": -2.251025438308716,
|
|
"logits/rejected": -1.9978622198104858,
|
|
"logps/chosen": -0.27270156145095825,
|
|
"logps/rejected": -0.283389151096344,
|
|
"loss": 0.36679738759994507,
|
|
"loss/core": 0.3456703722476959,
|
|
"loss/preference_sft": 0.27270156145095825,
|
|
"loss/reference_anchor": 1.0290827751159668,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 3.495182752609253,
|
|
"rewards/margins": 1.45361328125,
|
|
"rewards/rejected": 2.041569471359253,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31272217631340027,
|
|
"drift/rejected_abs": 0.15699851512908936,
|
|
"epoch": 0.3440243683927611,
|
|
"grad_norm": 99.5,
|
|
"ht_mnpo/logit": 0.15567512810230255,
|
|
"ht_mnpo/residual": 0.14817515015602112,
|
|
"ht_mnpo/residual_abs": 0.20475678145885468,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.879193898232725e-07,
|
|
"logits/chosen": -2.2675702571868896,
|
|
"logits/rejected": -2.1046242713928223,
|
|
"logps/chosen": -0.3180842399597168,
|
|
"logps/rejected": -0.28562289476394653,
|
|
"loss": 0.7316678762435913,
|
|
"loss/core": 0.7170267105102539,
|
|
"loss/preference_sft": 0.3180842399597168,
|
|
"loss/reference_anchor": 0.7002483606338501,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.126345157623291,
|
|
"rewards/margins": 1.5567514896392822,
|
|
"rewards/rejected": 1.5695935487747192,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3319775462150574,
|
|
"drift/rejected_abs": 0.18090906739234924,
|
|
"epoch": 0.3488024846204384,
|
|
"grad_norm": 1432.0,
|
|
"ht_mnpo/logit": 0.15248021483421326,
|
|
"ht_mnpo/residual": 0.14498022198677063,
|
|
"ht_mnpo/residual_abs": 0.21452391147613525,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8581310224609496e-07,
|
|
"logits/chosen": -2.4009974002838135,
|
|
"logits/rejected": -2.160431146621704,
|
|
"logps/chosen": -0.2676542401313782,
|
|
"logps/rejected": -0.2706208825111389,
|
|
"loss": 0.7889020442962646,
|
|
"loss/core": 0.7692867517471313,
|
|
"loss/preference_sft": 0.2676542401313782,
|
|
"loss/reference_anchor": 0.9540020823478699,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 3.3183226585388184,
|
|
"rewards/margins": 1.5248022079467773,
|
|
"rewards/rejected": 1.7935206890106201,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3430865705013275,
|
|
"drift/rejected_abs": 0.22682738304138184,
|
|
"epoch": 0.35358060084811566,
|
|
"grad_norm": 3.640625,
|
|
"ht_mnpo/logit": 0.1166459172964096,
|
|
"ht_mnpo/residual": 0.10914590209722519,
|
|
"ht_mnpo/residual_abs": 0.2188396006822586,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8368394534823635e-07,
|
|
"logits/chosen": -2.192263126373291,
|
|
"logits/rejected": -1.9854545593261719,
|
|
"logps/chosen": -0.28942209482192993,
|
|
"logps/rejected": -0.32335734367370605,
|
|
"loss": 0.8017171621322632,
|
|
"loss/core": 0.7787948846817017,
|
|
"loss/preference_sft": 0.28942209482192993,
|
|
"loss/reference_anchor": 1.1171759366989136,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.429896593093872,
|
|
"rewards/margins": 1.1664592027664185,
|
|
"rewards/rejected": 2.2634377479553223,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15755537152290344,
|
|
"drift/rejected_abs": 0.12303932011127472,
|
|
"epoch": 0.35835871707579287,
|
|
"grad_norm": 11.5,
|
|
"ht_mnpo/logit": 0.03453439846634865,
|
|
"ht_mnpo/residual": 0.027034390717744827,
|
|
"ht_mnpo/residual_abs": 0.11411330848932266,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8153271981852968e-07,
|
|
"logits/chosen": -2.3654236793518066,
|
|
"logits/rejected": -2.1134047508239746,
|
|
"logps/chosen": -0.26897451281547546,
|
|
"logps/rejected": -0.2847447991371155,
|
|
"loss": 0.10358532518148422,
|
|
"loss/core": 0.10106565803289413,
|
|
"loss/preference_sft": 0.26897451281547546,
|
|
"loss/reference_anchor": 0.09908583015203476,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.5745567083358765,
|
|
"rewards/margins": 0.3453438878059387,
|
|
"rewards/rejected": 1.2292128801345825,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25554612278938293,
|
|
"drift/rejected_abs": 0.20725803077220917,
|
|
"epoch": 0.36313683330347013,
|
|
"grad_norm": 39.0,
|
|
"ht_mnpo/logit": 0.048823170363903046,
|
|
"ht_mnpo/residual": 0.04132317006587982,
|
|
"ht_mnpo/residual_abs": 0.16914990544319153,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7936023464491812e-07,
|
|
"logits/chosen": -2.299844980239868,
|
|
"logits/rejected": -2.0326662063598633,
|
|
"logps/chosen": -0.3033023476600647,
|
|
"logps/rejected": -0.32690924406051636,
|
|
"loss": 0.3187548518180847,
|
|
"loss/core": 0.3046416640281677,
|
|
"loss/preference_sft": 0.3033023476600647,
|
|
"loss/reference_anchor": 0.6753290891647339,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.554044008255005,
|
|
"rewards/margins": 0.48823174834251404,
|
|
"rewards/rejected": 2.065812110900879,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19318945705890656,
|
|
"drift/rejected_abs": 0.09826628863811493,
|
|
"epoch": 0.36791494953114734,
|
|
"grad_norm": 23.75,
|
|
"ht_mnpo/logit": 0.09533454477787018,
|
|
"ht_mnpo/residual": 0.08783452957868576,
|
|
"ht_mnpo/residual_abs": 0.09699156880378723,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7716730681022723e-07,
|
|
"logits/chosen": -2.2895002365112305,
|
|
"logits/rejected": -2.017604351043701,
|
|
"logps/chosen": -0.2920383810997009,
|
|
"logps/rejected": -0.2935320734977722,
|
|
"loss": 0.07157917320728302,
|
|
"loss/core": 0.06837110221385956,
|
|
"loss/preference_sft": 0.2920383810997009,
|
|
"loss/reference_anchor": 0.1311999261379242,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.9301793575286865,
|
|
"rewards/margins": 0.9533454775810242,
|
|
"rewards/rejected": 0.9768339991569519,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.176864892244339,
|
|
"drift/rejected_abs": 0.11490440368652344,
|
|
"epoch": 0.3726930657588246,
|
|
"grad_norm": 62.5,
|
|
"ht_mnpo/logit": 0.07121428102254868,
|
|
"ht_mnpo/residual": 0.06371428817510605,
|
|
"ht_mnpo/residual_abs": 0.12565705180168152,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7495476098493152e-07,
|
|
"logits/chosen": -1.994593858718872,
|
|
"logits/rejected": -1.758146047592163,
|
|
"logps/chosen": -0.2808839976787567,
|
|
"logps/rejected": -0.3004426062107086,
|
|
"loss": 0.32486021518707275,
|
|
"loss/core": 0.3205881714820862,
|
|
"loss/preference_sft": 0.2808839976787567,
|
|
"loss/reference_anchor": 0.1855139434337616,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 1.767156958580017,
|
|
"rewards/margins": 0.712142825126648,
|
|
"rewards/rejected": 1.0550141334533691,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15783588588237762,
|
|
"drift/rejected_abs": 0.10911662876605988,
|
|
"epoch": 0.3774711819865018,
|
|
"grad_norm": 1.2265625,
|
|
"ht_mnpo/logit": 0.04880943149328232,
|
|
"ht_mnpo/residual": 0.04130943864583969,
|
|
"ht_mnpo/residual_abs": 0.06264761835336685,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7272342921702937e-07,
|
|
"logits/chosen": -2.422060966491699,
|
|
"logits/rejected": -2.158655881881714,
|
|
"logps/chosen": -0.26892244815826416,
|
|
"logps/rejected": -0.2726554572582245,
|
|
"loss": 0.026511918753385544,
|
|
"loss/core": 0.023896247148513794,
|
|
"loss/preference_sft": 0.26892244815826416,
|
|
"loss/reference_anchor": 0.10389119386672974,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5771405696868896,
|
|
"rewards/margins": 0.48809438943862915,
|
|
"rewards/rejected": 1.0890462398529053,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4108216166496277,
|
|
"drift/rejected_abs": 0.15476536750793457,
|
|
"epoch": 0.3822492982141791,
|
|
"grad_norm": 3.125,
|
|
"ht_mnpo/logit": 0.25611841678619385,
|
|
"ht_mnpo/residual": 0.24861836433410645,
|
|
"ht_mnpo/residual_abs": 0.2766857445240021,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.7047415061914393e-07,
|
|
"logits/chosen": -2.0853142738342285,
|
|
"logits/rejected": -1.8839391469955444,
|
|
"logps/chosen": -0.28590038418769836,
|
|
"logps/rejected": -0.30821120738983154,
|
|
"loss": 1.28208589553833,
|
|
"loss/core": 1.2619318962097168,
|
|
"loss/preference_sft": 0.28590038418769836,
|
|
"loss/reference_anchor": 0.9791082143783569,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.108216285705566,
|
|
"rewards/margins": 2.5611839294433594,
|
|
"rewards/rejected": 1.547032117843628,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16348758339881897,
|
|
"drift/rejected_abs": 0.11988557875156403,
|
|
"epoch": 0.3870274144418563,
|
|
"grad_norm": 0.953125,
|
|
"ht_mnpo/logit": 0.04386948049068451,
|
|
"ht_mnpo/residual": 0.03636947274208069,
|
|
"ht_mnpo/residual_abs": 0.06873659044504166,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6820777105296707e-07,
|
|
"logits/chosen": -2.353846788406372,
|
|
"logits/rejected": -2.104539394378662,
|
|
"logps/chosen": -0.2770633399486542,
|
|
"logps/rejected": -0.2738936245441437,
|
|
"loss": 0.0407264307141304,
|
|
"loss/core": 0.037368860095739365,
|
|
"loss/preference_sft": 0.2770633399486542,
|
|
"loss/reference_anchor": 0.14017218351364136,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6344867944717407,
|
|
"rewards/margins": 0.43869471549987793,
|
|
"rewards/rejected": 1.1957919597625732,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20585384964942932,
|
|
"drift/rejected_abs": 0.1777520775794983,
|
|
"epoch": 0.39180553066953355,
|
|
"grad_norm": 126.0,
|
|
"ht_mnpo/logit": 0.028088754042983055,
|
|
"ht_mnpo/residual": 0.020588750019669533,
|
|
"ht_mnpo/residual_abs": 0.1738569438457489,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6592514281116553e-07,
|
|
"logits/chosen": -2.227536678314209,
|
|
"logits/rejected": -2.0844969749450684,
|
|
"logps/chosen": -0.27204984426498413,
|
|
"logps/rejected": -0.2747117877006531,
|
|
"loss": 0.3877445161342621,
|
|
"loss/core": 0.38092032074928284,
|
|
"loss/preference_sft": 0.27204984426498413,
|
|
"loss/reference_anchor": 0.3140057921409607,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0569136142730713,
|
|
"rewards/margins": 0.2808874249458313,
|
|
"rewards/rejected": 1.7760260105133057,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16413165628910065,
|
|
"drift/rejected_abs": 0.13401861488819122,
|
|
"epoch": 0.39658364689721076,
|
|
"grad_norm": 33.5,
|
|
"ht_mnpo/logit": 0.030141586437821388,
|
|
"ht_mnpo/residual": 0.022641588002443314,
|
|
"ht_mnpo/residual_abs": 0.08408423513174057,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.636271242968684e-07,
|
|
"logits/chosen": -2.1277430057525635,
|
|
"logits/rejected": -1.9705203771591187,
|
|
"logps/chosen": -0.29165926575660706,
|
|
"logps/rejected": -0.2823432385921478,
|
|
"loss": 0.05560864135622978,
|
|
"loss/core": 0.05242307111620903,
|
|
"loss/preference_sft": 0.29165926575660706,
|
|
"loss/reference_anchor": 0.13011261820793152,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6413164138793945,
|
|
"rewards/margins": 0.30141589045524597,
|
|
"rewards/rejected": 1.3399007320404053,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25538012385368347,
|
|
"drift/rejected_abs": 0.09950107336044312,
|
|
"epoch": 0.401361763124888,
|
|
"grad_norm": 580.0,
|
|
"ht_mnpo/logit": 0.16329319775104523,
|
|
"ht_mnpo/residual": 0.1557932049036026,
|
|
"ht_mnpo/residual_abs": 0.19033615291118622,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6131457970085684e-07,
|
|
"logits/chosen": -2.0850791931152344,
|
|
"logits/rejected": -1.8701903820037842,
|
|
"logps/chosen": -0.28022292256355286,
|
|
"logps/rejected": -0.2937292456626892,
|
|
"loss": 0.26428142189979553,
|
|
"loss/core": 0.26025694608688354,
|
|
"loss/preference_sft": 0.28022292256355286,
|
|
"loss/reference_anchor": 0.17319951951503754,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.552206516265869,
|
|
"rewards/margins": 1.6329319477081299,
|
|
"rewards/rejected": 0.9192743301391602,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22731247544288635,
|
|
"drift/rejected_abs": 0.19945786893367767,
|
|
"epoch": 0.40613987935256524,
|
|
"grad_norm": 0.60546875,
|
|
"ht_mnpo/logit": 0.02869640663266182,
|
|
"ht_mnpo/residual": 0.021196408197283745,
|
|
"ht_mnpo/residual_abs": 0.17983601987361908,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5898837867657727e-07,
|
|
"logits/chosen": -2.156146764755249,
|
|
"logits/rejected": -1.908090353012085,
|
|
"logps/chosen": -0.29179322719573975,
|
|
"logps/rejected": -0.3070332705974579,
|
|
"loss": 0.43886813521385193,
|
|
"loss/core": 0.4307543635368347,
|
|
"loss/preference_sft": 0.29179322719573975,
|
|
"loss/reference_anchor": 0.3765077292919159,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2724695205688477,
|
|
"rewards/margins": 0.2869641184806824,
|
|
"rewards/rejected": 1.9855053424835205,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26135116815567017,
|
|
"drift/rejected_abs": 0.1429075300693512,
|
|
"epoch": 0.4109179955802425,
|
|
"grad_norm": 432.0,
|
|
"ht_mnpo/logit": 0.1191127672791481,
|
|
"ht_mnpo/residual": 0.11161275207996368,
|
|
"ht_mnpo/residual_abs": 0.13684937357902527,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5664939601310023e-07,
|
|
"logits/chosen": -2.198462963104248,
|
|
"logits/rejected": -1.987441062927246,
|
|
"logps/chosen": -0.2752520442008972,
|
|
"logps/rejected": -0.28098762035369873,
|
|
"loss": 0.267617791891098,
|
|
"loss/core": 0.25560644268989563,
|
|
"loss/preference_sft": 0.2752520442008972,
|
|
"loss/reference_anchor": 0.5730425119400024,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6121487617492676,
|
|
"rewards/margins": 1.1911275386810303,
|
|
"rewards/rejected": 1.4210212230682373,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1936320811510086,
|
|
"drift/rejected_abs": 0.11375801265239716,
|
|
"epoch": 0.4156961118079197,
|
|
"grad_norm": 11.6875,
|
|
"ht_mnpo/logit": 0.08014325797557831,
|
|
"ht_mnpo/residual": 0.07264326512813568,
|
|
"ht_mnpo/residual_abs": 0.09690365940332413,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5429851130614807e-07,
|
|
"logits/chosen": -2.1249842643737793,
|
|
"logits/rejected": -1.8592697381973267,
|
|
"logps/chosen": -0.3011928200721741,
|
|
"logps/rejected": -0.31247323751449585,
|
|
"loss": 0.05613823980093002,
|
|
"loss/core": 0.05347650125622749,
|
|
"loss/preference_sft": 0.3011928200721741,
|
|
"loss/reference_anchor": 0.10296765714883804,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.9342870712280273,
|
|
"rewards/margins": 0.8014326095581055,
|
|
"rewards/rejected": 1.1328542232513428,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2973514199256897,
|
|
"drift/rejected_abs": 0.10415633022785187,
|
|
"epoch": 0.420474228035597,
|
|
"grad_norm": 476.0,
|
|
"ht_mnpo/logit": 0.22436299920082092,
|
|
"ht_mnpo/residual": 0.21686296164989471,
|
|
"ht_mnpo/residual_abs": 0.2422914057970047,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.51936608627315e-07,
|
|
"logits/chosen": -2.127042293548584,
|
|
"logits/rejected": -1.8621797561645508,
|
|
"logps/chosen": -0.30403512716293335,
|
|
"logps/rejected": -0.3232555389404297,
|
|
"loss": 1.0134453773498535,
|
|
"loss/core": 1.0037100315093994,
|
|
"loss/preference_sft": 0.30403512716293335,
|
|
"loss/reference_anchor": 0.4563591480255127,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.9734644889831543,
|
|
"rewards/margins": 2.2436299324035645,
|
|
"rewards/rejected": 0.7298344373703003,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19231143593788147,
|
|
"drift/rejected_abs": 0.09996972978115082,
|
|
"epoch": 0.4252523442632742,
|
|
"grad_norm": 31.875,
|
|
"ht_mnpo/logit": 0.09261570125818253,
|
|
"ht_mnpo/residual": 0.0851157084107399,
|
|
"ht_mnpo/residual_abs": 0.09635229408740997,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.4956457619160375e-07,
|
|
"logits/chosen": -2.433109760284424,
|
|
"logits/rejected": -2.0341031551361084,
|
|
"logps/chosen": -0.29921525716781616,
|
|
"logps/rejected": -0.3007660508155823,
|
|
"loss": 0.08675873279571533,
|
|
"loss/core": 0.0813307985663414,
|
|
"loss/preference_sft": 0.29921525716781616,
|
|
"loss/reference_anchor": 0.2414754182100296,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9225269556045532,
|
|
"rewards/margins": 0.9261569976806641,
|
|
"rewards/rejected": 0.9963698387145996,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15865620970726013,
|
|
"drift/rejected_abs": 0.11887378990650177,
|
|
"epoch": 0.43003046049095145,
|
|
"grad_norm": 152.0,
|
|
"ht_mnpo/logit": 0.0395728163421154,
|
|
"ht_mnpo/residual": 0.03207281976938248,
|
|
"ht_mnpo/residual_abs": 0.12159354984760284,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.471833060234044e-07,
|
|
"logits/chosen": -2.210747241973877,
|
|
"logits/rejected": -1.945389986038208,
|
|
"logps/chosen": -0.2810204029083252,
|
|
"logps/rejected": -0.30275437235832214,
|
|
"loss": 0.1328326165676117,
|
|
"loss/core": 0.129548579454422,
|
|
"loss/preference_sft": 0.2810204029083252,
|
|
"loss/reference_anchor": 0.1361009180545807,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.5836235284805298,
|
|
"rewards/margins": 0.3957282602787018,
|
|
"rewards/rejected": 1.1878951787948608,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25045090913772583,
|
|
"drift/rejected_abs": 0.13791918754577637,
|
|
"epoch": 0.43480857671862866,
|
|
"grad_norm": 100.0,
|
|
"ht_mnpo/logit": 0.112596794962883,
|
|
"ht_mnpo/residual": 0.10509680211544037,
|
|
"ht_mnpo/residual_abs": 0.15906308591365814,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.4479369362104037e-07,
|
|
"logits/chosen": -2.2764220237731934,
|
|
"logits/rejected": -2.0353825092315674,
|
|
"logps/chosen": -0.29567039012908936,
|
|
"logps/rejected": -0.30568239092826843,
|
|
"loss": 0.3157535195350647,
|
|
"loss/core": 0.30716970562934875,
|
|
"loss/preference_sft": 0.29567039012908936,
|
|
"loss/reference_anchor": 0.3996273875236511,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.50372576713562,
|
|
"rewards/margins": 1.1259678602218628,
|
|
"rewards/rejected": 1.3777577877044678,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1878124624490738,
|
|
"drift/rejected_abs": 0.14636285603046417,
|
|
"epoch": 0.4395866929463059,
|
|
"grad_norm": 3.53125,
|
|
"ht_mnpo/logit": 0.04185149818658829,
|
|
"ht_mnpo/residual": 0.03435150161385536,
|
|
"ht_mnpo/residual_abs": 0.07132266461849213,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.4239663762000817e-07,
|
|
"logits/chosen": -2.317431688308716,
|
|
"logits/rejected": -2.0158653259277344,
|
|
"logps/chosen": -0.2637852132320404,
|
|
"logps/rejected": -0.256014883518219,
|
|
"loss": 0.026135962456464767,
|
|
"loss/core": 0.023039747029542923,
|
|
"loss/preference_sft": 0.2637852132320404,
|
|
"loss/reference_anchor": 0.1284322738647461,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.878124475479126,
|
|
"rewards/margins": 0.41851502656936646,
|
|
"rewards/rejected": 1.4596095085144043,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2502024471759796,
|
|
"drift/rejected_abs": 0.16358964145183563,
|
|
"epoch": 0.44436480917398313,
|
|
"grad_norm": 17.75,
|
|
"ht_mnpo/logit": 0.0867425948381424,
|
|
"ht_mnpo/residual": 0.07924258708953857,
|
|
"ht_mnpo/residual_abs": 0.15044549107551575,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3999303945503747e-07,
|
|
"logits/chosen": -2.1240248680114746,
|
|
"logits/rejected": -1.9239466190338135,
|
|
"logps/chosen": -0.31266671419143677,
|
|
"logps/rejected": -0.3123919367790222,
|
|
"loss": 0.19001127779483795,
|
|
"loss/core": 0.183534175157547,
|
|
"loss/preference_sft": 0.31266671419143677,
|
|
"loss/reference_anchor": 0.2925878167152405,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.501178741455078,
|
|
"rewards/margins": 0.8674259185791016,
|
|
"rewards/rejected": 1.6337528228759766,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10216370970010757,
|
|
"drift/rejected_abs": 0.12402309477329254,
|
|
"epoch": 0.4491429254016604,
|
|
"grad_norm": 117.0,
|
|
"ht_mnpo/logit": -0.02179349772632122,
|
|
"ht_mnpo/residual": -0.029293496161699295,
|
|
"ht_mnpo/residual_abs": 0.09518697112798691,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3758380302109808e-07,
|
|
"logits/chosen": -2.449554681777954,
|
|
"logits/rejected": -2.152750015258789,
|
|
"logps/chosen": -0.27131548523902893,
|
|
"logps/rejected": -0.2756040394306183,
|
|
"loss": 0.09835384786128998,
|
|
"loss/core": 0.09614761918783188,
|
|
"loss/preference_sft": 0.27131548523902893,
|
|
"loss/reference_anchor": 0.0831795483827591,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.021196961402893,
|
|
"rewards/margins": -0.21793493628501892,
|
|
"rewards/rejected": 1.2391319274902344,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15605005621910095,
|
|
"drift/rejected_abs": 0.09966011345386505,
|
|
"epoch": 0.4539210416293376,
|
|
"grad_norm": 86.0,
|
|
"ht_mnpo/logit": 0.05629577487707138,
|
|
"ht_mnpo/residual": 0.048795782029628754,
|
|
"ht_mnpo/residual_abs": 0.1054389700293541,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.351698343334823e-07,
|
|
"logits/chosen": -2.2466909885406494,
|
|
"logits/rejected": -1.9857765436172485,
|
|
"logps/chosen": -0.2887324392795563,
|
|
"logps/rejected": -0.28986722230911255,
|
|
"loss": 0.1072268858551979,
|
|
"loss/core": 0.10453776270151138,
|
|
"loss/preference_sft": 0.2887324392795563,
|
|
"loss/reference_anchor": 0.10558246076107025,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.5577256679534912,
|
|
"rewards/margins": 0.5629578828811646,
|
|
"rewards/rejected": 0.9947676658630371,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1687658131122589,
|
|
"drift/rejected_abs": 0.13127224147319794,
|
|
"epoch": 0.45869915785701487,
|
|
"grad_norm": 2.265625,
|
|
"ht_mnpo/logit": 0.03775458782911301,
|
|
"ht_mnpo/residual": 0.030254587531089783,
|
|
"ht_mnpo/residual_abs": 0.09340941905975342,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3275204118708942e-07,
|
|
"logits/chosen": -2.268664598464966,
|
|
"logits/rejected": -2.0399532318115234,
|
|
"logps/chosen": -0.27916237711906433,
|
|
"logps/rejected": -0.2935280203819275,
|
|
"loss": 0.08096377551555634,
|
|
"loss/core": 0.0781911164522171,
|
|
"loss/preference_sft": 0.27916237711906433,
|
|
"loss/reference_anchor": 0.11071737110614777,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.6864219903945923,
|
|
"rewards/margins": 0.37754589319229126,
|
|
"rewards/rejected": 1.3088760375976562,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22259202599525452,
|
|
"drift/rejected_abs": 0.16338473558425903,
|
|
"epoch": 0.46347727408469214,
|
|
"grad_norm": 21.875,
|
|
"ht_mnpo/logit": 0.05938667058944702,
|
|
"ht_mnpo/residual": 0.0518866702914238,
|
|
"ht_mnpo/residual_abs": 0.1358523964881897,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3033133281504132e-07,
|
|
"logits/chosen": -2.1293797492980957,
|
|
"logits/rejected": -1.8755149841308594,
|
|
"logps/chosen": -0.2896140217781067,
|
|
"logps/rejected": -0.2774605453014374,
|
|
"loss": 0.21404489874839783,
|
|
"loss/core": 0.20932647585868835,
|
|
"loss/preference_sft": 0.2896140217781067,
|
|
"loss/reference_anchor": 0.2069588154554367,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.225459098815918,
|
|
"rewards/margins": 0.5938667058944702,
|
|
"rewards/rejected": 1.6315925121307373,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24862901866436005,
|
|
"drift/rejected_abs": 0.13741818070411682,
|
|
"epoch": 0.46825539031236935,
|
|
"grad_norm": 14.125,
|
|
"ht_mnpo/logit": 0.11346457898616791,
|
|
"ht_mnpo/residual": 0.10596457868814468,
|
|
"ht_mnpo/residual_abs": 0.14489370584487915,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2790861954675702e-07,
|
|
"logits/chosen": -2.007351875305176,
|
|
"logits/rejected": -1.7956119775772095,
|
|
"logps/chosen": -0.2839955687522888,
|
|
"logps/rejected": -0.28634992241859436,
|
|
"loss": 0.21525898575782776,
|
|
"loss/core": 0.21072621643543243,
|
|
"loss/preference_sft": 0.2839955687522888,
|
|
"loss/reference_anchor": 0.19823768734931946,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.4841885566711426,
|
|
"rewards/margins": 1.1346458196640015,
|
|
"rewards/rejected": 1.3495427370071411,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14347688853740692,
|
|
"drift/rejected_abs": 0.09557531028985977,
|
|
"epoch": 0.4730335065400466,
|
|
"grad_norm": 16.375,
|
|
"ht_mnpo/logit": 0.048013053834438324,
|
|
"ht_mnpo/residual": 0.0405130572617054,
|
|
"ht_mnpo/residual_abs": 0.09559812396764755,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2548481246561504e-07,
|
|
"logits/chosen": -2.1509878635406494,
|
|
"logits/rejected": -1.9557958841323853,
|
|
"logps/chosen": -0.29566025733947754,
|
|
"logps/rejected": -0.31424885988235474,
|
|
"loss": 0.09365914762020111,
|
|
"loss/core": 0.09119492024183273,
|
|
"loss/preference_sft": 0.29566025733947754,
|
|
"loss/reference_anchor": 0.09364519268274307,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.4331740140914917,
|
|
"rewards/margins": 0.48013052344322205,
|
|
"rewards/rejected": 0.9530435800552368,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2629546821117401,
|
|
"drift/rejected_abs": 0.2393544614315033,
|
|
"epoch": 0.4778116227677238,
|
|
"grad_norm": 68.5,
|
|
"ht_mnpo/logit": 0.024394646286964417,
|
|
"ht_mnpo/residual": 0.016894647851586342,
|
|
"ht_mnpo/residual_abs": 0.20083613693714142,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.230608230663321e-07,
|
|
"logits/chosen": -2.3212738037109375,
|
|
"logits/rejected": -2.0327229499816895,
|
|
"logps/chosen": -0.27783674001693726,
|
|
"logps/rejected": -0.2940555810928345,
|
|
"loss": 0.32842302322387695,
|
|
"loss/core": 0.31986716389656067,
|
|
"loss/preference_sft": 0.27783674001693726,
|
|
"loss/reference_anchor": 0.40000849962234497,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.6291868686676025,
|
|
"rewards/margins": 0.24394643306732178,
|
|
"rewards/rejected": 2.385240316390991,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2415897399187088,
|
|
"drift/rejected_abs": 0.13792410492897034,
|
|
"epoch": 0.4825897389954011,
|
|
"grad_norm": 36.75,
|
|
"ht_mnpo/logit": 0.10351978242397308,
|
|
"ht_mnpo/residual": 0.09601978212594986,
|
|
"ht_mnpo/residual_abs": 0.17322488129138947,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.206375629121874e-07,
|
|
"logits/chosen": -2.1131768226623535,
|
|
"logits/rejected": -1.8549270629882812,
|
|
"logps/chosen": -0.2813659906387329,
|
|
"logps/rejected": -0.28420156240463257,
|
|
"loss": 0.38091525435447693,
|
|
"loss/core": 0.37360432744026184,
|
|
"loss/preference_sft": 0.2813659906387329,
|
|
"loss/reference_anchor": 0.33741098642349243,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.4129114151000977,
|
|
"rewards/margins": 1.0351978540420532,
|
|
"rewards/rejected": 1.3777135610580444,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18775442242622375,
|
|
"drift/rejected_abs": 0.1655738800764084,
|
|
"epoch": 0.4873678552230783,
|
|
"grad_norm": 2.984375,
|
|
"ht_mnpo/logit": 0.022700464352965355,
|
|
"ht_mnpo/residual": 0.015200458467006683,
|
|
"ht_mnpo/residual_abs": 0.09576776623725891,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1821594329222079e-07,
|
|
"logits/chosen": -2.3727898597717285,
|
|
"logits/rejected": -2.067782402038574,
|
|
"logps/chosen": -0.27853554487228394,
|
|
"logps/rejected": -0.296770304441452,
|
|
"loss": 0.06544214487075806,
|
|
"loss/core": 0.06130026653409004,
|
|
"loss/preference_sft": 0.27853554487228394,
|
|
"loss/reference_anchor": 0.17924094200134277,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.8766510486602783,
|
|
"rewards/margins": 0.22700461745262146,
|
|
"rewards/rejected": 1.649646520614624,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2205163985490799,
|
|
"drift/rejected_abs": 0.15725180506706238,
|
|
"epoch": 0.49214597145075556,
|
|
"grad_norm": 0.453125,
|
|
"ht_mnpo/logit": 0.06345703452825546,
|
|
"ht_mnpo/residual": 0.05595703050494194,
|
|
"ht_mnpo/residual_abs": 0.1105417013168335,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.157968748785344e-07,
|
|
"logits/chosen": -2.234668016433716,
|
|
"logits/rejected": -1.8898760080337524,
|
|
"logps/chosen": -0.28748154640197754,
|
|
"logps/rejected": -0.2847779393196106,
|
|
"loss": 0.09203025698661804,
|
|
"loss/core": 0.08730658143758774,
|
|
"loss/preference_sft": 0.28748154640197754,
|
|
"loss/reference_anchor": 0.20743636786937714,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.2042317390441895,
|
|
"rewards/margins": 0.634570300579071,
|
|
"rewards/rejected": 1.5696613788604736,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27758151292800903,
|
|
"drift/rejected_abs": 0.15809740126132965,
|
|
"epoch": 0.49692408767843277,
|
|
"grad_norm": 36.5,
|
|
"ht_mnpo/logit": 0.11959369480609894,
|
|
"ht_mnpo/residual": 0.11209367215633392,
|
|
"ht_mnpo/residual_abs": 0.2109643965959549,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1338126738382597e-07,
|
|
"logits/chosen": -2.270226001739502,
|
|
"logits/rejected": -2.025266170501709,
|
|
"logps/chosen": -0.2758481800556183,
|
|
"logps/rejected": -0.26102808117866516,
|
|
"loss": 0.30472052097320557,
|
|
"loss/core": 0.2978774905204773,
|
|
"loss/preference_sft": 0.2758481800556183,
|
|
"loss/reference_anchor": 0.31456848978996277,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7751450538635254,
|
|
"rewards/margins": 1.195936918258667,
|
|
"rewards/rejected": 1.5792083740234375,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2691987454891205,
|
|
"drift/rejected_abs": 0.15224085748195648,
|
|
"epoch": 0.50170220390611,
|
|
"grad_norm": 207.0,
|
|
"ht_mnpo/logit": 0.12795712053775787,
|
|
"ht_mnpo/residual": 0.12045712769031525,
|
|
"ht_mnpo/residual_abs": 0.2664859890937805,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1097002921928316e-07,
|
|
"logits/chosen": -2.1398863792419434,
|
|
"logits/rejected": -1.8041080236434937,
|
|
"logps/chosen": -0.30692705512046814,
|
|
"logps/rejected": -0.33052974939346313,
|
|
"loss": 0.7266106009483337,
|
|
"loss/core": 0.718201756477356,
|
|
"loss/preference_sft": 0.30692705512046814,
|
|
"loss/reference_anchor": 0.3897519111633301,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.6892130374908447,
|
|
"rewards/margins": 1.279571294784546,
|
|
"rewards/rejected": 1.409641981124878,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25865083932876587,
|
|
"drift/rejected_abs": 0.0813761055469513,
|
|
"epoch": 0.5064803201337873,
|
|
"grad_norm": 19.75,
|
|
"ht_mnpo/logit": 0.1827838122844696,
|
|
"ht_mnpo/residual": 0.17528381943702698,
|
|
"ht_mnpo/residual_abs": 0.19191543757915497,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0856406715296717e-07,
|
|
"logits/chosen": -2.2501230239868164,
|
|
"logits/rejected": -1.9791781902313232,
|
|
"logps/chosen": -0.2921066880226135,
|
|
"logps/rejected": -0.30198463797569275,
|
|
"loss": 0.7703770399093628,
|
|
"loss/core": 0.7617815136909485,
|
|
"loss/preference_sft": 0.2921066880226135,
|
|
"loss/reference_anchor": 0.4005637764930725,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.5850157737731934,
|
|
"rewards/margins": 1.8278381824493408,
|
|
"rewards/rejected": 0.7571774125099182,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21662993729114532,
|
|
"drift/rejected_abs": 0.2024078369140625,
|
|
"epoch": 0.5112584363614645,
|
|
"grad_norm": 3.640625,
|
|
"ht_mnpo/logit": 0.014330679550766945,
|
|
"ht_mnpo/residual": 0.006830704398453236,
|
|
"ht_mnpo/residual_abs": 0.24278083443641663,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.061642859688146e-07,
|
|
"logits/chosen": -2.3559963703155518,
|
|
"logits/rejected": -2.213176727294922,
|
|
"logps/chosen": -0.31593745946884155,
|
|
"logps/rejected": -0.317546546459198,
|
|
"loss": 1.2009260654449463,
|
|
"loss/core": 1.1859214305877686,
|
|
"loss/preference_sft": 0.31593745946884155,
|
|
"loss/reference_anchor": 0.7186298370361328,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1661620140075684,
|
|
"rewards/margins": 0.1433069258928299,
|
|
"rewards/rejected": 2.022855281829834,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19005665183067322,
|
|
"drift/rejected_abs": 0.07541195303201675,
|
|
"epoch": 0.5160365525891417,
|
|
"grad_norm": 6.3125,
|
|
"ht_mnpo/logit": 0.11505275964736938,
|
|
"ht_mnpo/residual": 0.10755278170108795,
|
|
"ht_mnpo/residual_abs": 0.13360702991485596,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0377158812638491e-07,
|
|
"logits/chosen": -2.269131898880005,
|
|
"logits/rejected": -2.05694317817688,
|
|
"logps/chosen": -0.3238433599472046,
|
|
"logps/rejected": -0.3159184157848358,
|
|
"loss": 0.26619070768356323,
|
|
"loss/core": 0.2617434561252594,
|
|
"loss/preference_sft": 0.3238433599472046,
|
|
"loss/reference_anchor": 0.18997910618782043,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9001648426055908,
|
|
"rewards/margins": 1.1505277156829834,
|
|
"rewards/rejected": 0.7496371865272522,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2369406670331955,
|
|
"drift/rejected_abs": 0.0789075642824173,
|
|
"epoch": 0.5208146688168189,
|
|
"grad_norm": 13.875,
|
|
"ht_mnpo/logit": 0.15915516018867493,
|
|
"ht_mnpo/residual": 0.1516551524400711,
|
|
"ht_mnpo/residual_abs": 0.1720009595155716,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0138687342148249e-07,
|
|
"logits/chosen": -2.434389352798462,
|
|
"logits/rejected": -2.192600727081299,
|
|
"logps/chosen": -0.28083503246307373,
|
|
"logps/rejected": -0.2732694745063782,
|
|
"loss": 0.35217469930648804,
|
|
"loss/core": 0.34702593088150024,
|
|
"loss/preference_sft": 0.28083503246307373,
|
|
"loss/reference_anchor": 0.229354590177536,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3685014247894287,
|
|
"rewards/margins": 1.5915515422821045,
|
|
"rewards/rejected": 0.7769501209259033,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2724258005619049,
|
|
"drift/rejected_abs": 0.12468817085027695,
|
|
"epoch": 0.5255927850444962,
|
|
"grad_norm": 2.078125,
|
|
"ht_mnpo/logit": 0.14971444010734558,
|
|
"ht_mnpo/residual": 0.14221441745758057,
|
|
"ht_mnpo/residual_abs": 0.1590646654367447,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.90110386477801e-08,
|
|
"logits/chosen": -2.0379040241241455,
|
|
"logits/rejected": -1.7684564590454102,
|
|
"logps/chosen": -0.3047271966934204,
|
|
"logps/rejected": -0.3073706328868866,
|
|
"loss": 0.3628101348876953,
|
|
"loss/core": 0.3507049083709717,
|
|
"loss/preference_sft": 0.3047271966934204,
|
|
"loss/reference_anchor": 0.5747870206832886,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.7242579460144043,
|
|
"rewards/margins": 1.497144341468811,
|
|
"rewards/rejected": 1.227113962173462,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31042712926864624,
|
|
"drift/rejected_abs": 0.18171672523021698,
|
|
"epoch": 0.5303709012721735,
|
|
"grad_norm": 1568.0,
|
|
"ht_mnpo/logit": 0.12890593707561493,
|
|
"ht_mnpo/residual": 0.1214059591293335,
|
|
"ht_mnpo/residual_abs": 0.24321047961711884,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.664497725957164e-08,
|
|
"logits/chosen": -2.0988707542419434,
|
|
"logits/rejected": -1.9185583591461182,
|
|
"logps/chosen": -0.2986465096473694,
|
|
"logps/rejected": -0.29767924547195435,
|
|
"loss": 1.2037521600723267,
|
|
"loss/core": 1.1875665187835693,
|
|
"loss/preference_sft": 0.2986465096473694,
|
|
"loss/reference_anchor": 0.7794210314750671,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.103039026260376,
|
|
"rewards/margins": 1.2890592813491821,
|
|
"rewards/rejected": 1.8139795064926147,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19171099364757538,
|
|
"drift/rejected_abs": 0.10674883425235748,
|
|
"epoch": 0.5351490174998507,
|
|
"grad_norm": 0.10498046875,
|
|
"ht_mnpo/logit": 0.08548573404550552,
|
|
"ht_mnpo/residual": 0.0779857337474823,
|
|
"ht_mnpo/residual_abs": 0.10989482700824738,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.428957903578045e-08,
|
|
"logits/chosen": -2.2540018558502197,
|
|
"logits/rejected": -2.0119833946228027,
|
|
"logps/chosen": -0.26887863874435425,
|
|
"logps/rejected": -0.26674965023994446,
|
|
"loss": 0.09468104690313339,
|
|
"loss/core": 0.09218241274356842,
|
|
"loss/preference_sft": 0.26887863874435425,
|
|
"loss/reference_anchor": 0.09804411232471466,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.915593147277832,
|
|
"rewards/margins": 0.8548572659492493,
|
|
"rewards/rejected": 1.060735821723938,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2377406656742096,
|
|
"drift/rejected_abs": 0.20291483402252197,
|
|
"epoch": 0.5399271337275279,
|
|
"grad_norm": 51.25,
|
|
"ht_mnpo/logit": 0.0349448099732399,
|
|
"ht_mnpo/residual": 0.027444809675216675,
|
|
"ht_mnpo/residual_abs": 0.16346418857574463,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.194572974534976e-08,
|
|
"logits/chosen": -1.9928922653198242,
|
|
"logits/rejected": -1.8542598485946655,
|
|
"logps/chosen": -0.2798479497432709,
|
|
"logps/rejected": -0.2841023802757263,
|
|
"loss": 0.23373666405677795,
|
|
"loss/core": 0.22751207649707794,
|
|
"loss/preference_sft": 0.2798479497432709,
|
|
"loss/reference_anchor": 0.2832440733909607,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.37677001953125,
|
|
"rewards/margins": 0.3494481146335602,
|
|
"rewards/rejected": 2.0273215770721436,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15430274605751038,
|
|
"drift/rejected_abs": 0.1468575894832611,
|
|
"epoch": 0.5447052499552052,
|
|
"grad_norm": 40.75,
|
|
"ht_mnpo/logit": 0.0075254677794873714,
|
|
"ht_mnpo/residual": 2.5472789275227115e-05,
|
|
"ht_mnpo/residual_abs": 0.1423179656267166,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.96143108141412e-08,
|
|
"logits/chosen": -2.259716749191284,
|
|
"logits/rejected": -2.0335276126861572,
|
|
"logps/chosen": -0.3092796802520752,
|
|
"logps/rejected": -0.30099526047706604,
|
|
"loss": 0.22612264752388,
|
|
"loss/core": 0.22155435383319855,
|
|
"loss/preference_sft": 0.3092796802520752,
|
|
"loss/reference_anchor": 0.19748570024967194,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.5426218509674072,
|
|
"rewards/margins": 0.0752546414732933,
|
|
"rewards/rejected": 1.46736741065979,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16749927401542664,
|
|
"drift/rejected_abs": 0.10397598892450333,
|
|
"epoch": 0.5494833661828824,
|
|
"grad_norm": 16.375,
|
|
"ht_mnpo/logit": 0.06339401006698608,
|
|
"ht_mnpo/residual": 0.05589400604367256,
|
|
"ht_mnpo/residual_abs": 0.06961973011493683,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.72961989934668e-08,
|
|
"logits/chosen": -2.3487277030944824,
|
|
"logits/rejected": -2.1463935375213623,
|
|
"logps/chosen": -0.26586031913757324,
|
|
"logps/rejected": -0.27124351263046265,
|
|
"loss": 0.035394370555877686,
|
|
"loss/core": 0.033535875380039215,
|
|
"loss/preference_sft": 0.26586031913757324,
|
|
"loss/reference_anchor": 0.06633836030960083,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.6726531982421875,
|
|
"rewards/margins": 0.6339401006698608,
|
|
"rewards/rejected": 1.0387132167816162,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17038491368293762,
|
|
"drift/rejected_abs": 0.12820149958133698,
|
|
"epoch": 0.5542614824105596,
|
|
"grad_norm": 239.0,
|
|
"ht_mnpo/logit": 0.04234672710299492,
|
|
"ht_mnpo/residual": 0.03484673053026199,
|
|
"ht_mnpo/residual_abs": 0.13217854499816895,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.499226603037854e-08,
|
|
"logits/chosen": -2.1979475021362305,
|
|
"logits/rejected": -1.981332540512085,
|
|
"logps/chosen": -0.3051069676876068,
|
|
"logps/rejected": -0.30111053586006165,
|
|
"loss": 0.24051013588905334,
|
|
"loss/core": 0.23614850640296936,
|
|
"loss/preference_sft": 0.3051069676876068,
|
|
"loss/reference_anchor": 0.18757155537605286,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.7025823593139648,
|
|
"rewards/margins": 0.4234672486782074,
|
|
"rewards/rejected": 1.2791149616241455,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13287939131259918,
|
|
"drift/rejected_abs": 0.12429249286651611,
|
|
"epoch": 0.5590395986382368,
|
|
"grad_norm": 9.375,
|
|
"ht_mnpo/logit": 0.008631383068859577,
|
|
"ht_mnpo/residual": 0.0011313877766951919,
|
|
"ht_mnpo/residual_abs": 0.11924134194850922,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.270337833983987e-08,
|
|
"logits/chosen": -2.3649332523345947,
|
|
"logits/rejected": -2.1235404014587402,
|
|
"logps/chosen": -0.2820286154747009,
|
|
"logps/rejected": -0.30536872148513794,
|
|
"loss": 0.13413414359092712,
|
|
"loss/core": 0.13124492764472961,
|
|
"loss/preference_sft": 0.2820286154747009,
|
|
"loss/reference_anchor": 0.11625870317220688,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.328194260597229,
|
|
"rewards/margins": 0.08631384372711182,
|
|
"rewards/rejected": 1.2418802976608276,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25080275535583496,
|
|
"drift/rejected_abs": 0.15849502384662628,
|
|
"epoch": 0.5638177148659141,
|
|
"grad_norm": 134.0,
|
|
"ht_mnpo/logit": 0.0924316793680191,
|
|
"ht_mnpo/residual": 0.08493169397115707,
|
|
"ht_mnpo/residual_abs": 0.20355942845344543,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.04303966789025e-08,
|
|
"logits/chosen": -2.183878183364868,
|
|
"logits/rejected": -2.0252556800842285,
|
|
"logps/chosen": -0.2765917181968689,
|
|
"logps/rejected": -0.27511608600616455,
|
|
"loss": 0.5015133619308472,
|
|
"loss/core": 0.49359574913978577,
|
|
"loss/preference_sft": 0.2765917181968689,
|
|
"loss/reference_anchor": 0.36821773648262024,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5080275535583496,
|
|
"rewards/margins": 0.9243170022964478,
|
|
"rewards/rejected": 1.5837104320526123,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23994091153144836,
|
|
"drift/rejected_abs": 0.13920912146568298,
|
|
"epoch": 0.5685958310935914,
|
|
"grad_norm": 76.0,
|
|
"ht_mnpo/logit": 0.100993812084198,
|
|
"ht_mnpo/residual": 0.09349381923675537,
|
|
"ht_mnpo/residual_abs": 0.15858511626720428,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.817417582301098e-08,
|
|
"logits/chosen": -2.1511635780334473,
|
|
"logits/rejected": -2.000541925430298,
|
|
"logps/chosen": -0.28564023971557617,
|
|
"logps/rejected": -0.2844979763031006,
|
|
"loss": 0.19406889379024506,
|
|
"loss/core": 0.19000914692878723,
|
|
"loss/preference_sft": 0.28564023971557617,
|
|
"loss/reference_anchor": 0.174424409866333,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.398890972137451,
|
|
"rewards/margins": 1.0099382400512695,
|
|
"rewards/rejected": 1.3889529705047607,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1661892980337143,
|
|
"drift/rejected_abs": 0.14502601325511932,
|
|
"epoch": 0.5733739473212686,
|
|
"grad_norm": 1.375,
|
|
"ht_mnpo/logit": 0.022466612979769707,
|
|
"ht_mnpo/residual": 0.014966619201004505,
|
|
"ht_mnpo/residual_abs": 0.10731730610132217,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.59355642445566e-08,
|
|
"logits/chosen": -2.216158628463745,
|
|
"logits/rejected": -2.082313060760498,
|
|
"logps/chosen": -0.2715299725532532,
|
|
"logps/rejected": -0.27377790212631226,
|
|
"loss": 0.13416652381420135,
|
|
"loss/core": 0.130991131067276,
|
|
"loss/preference_sft": 0.2715299725532532,
|
|
"loss/reference_anchor": 0.13161684572696686,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6614080667495728,
|
|
"rewards/margins": 0.22466608881950378,
|
|
"rewards/rejected": 1.4367419481277466,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1707453578710556,
|
|
"drift/rejected_abs": 0.1772371083498001,
|
|
"epoch": 0.5781520635489459,
|
|
"grad_norm": 524.0,
|
|
"ht_mnpo/logit": -0.006207427475601435,
|
|
"ht_mnpo/residual": -0.013707419857382774,
|
|
"ht_mnpo/residual_abs": 0.13802742958068848,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.37154037938015e-08,
|
|
"logits/chosen": -2.2585723400115967,
|
|
"logits/rejected": -1.9065723419189453,
|
|
"logps/chosen": -0.2895175814628601,
|
|
"logps/rejected": -0.3101463317871094,
|
|
"loss": 0.2447996586561203,
|
|
"loss/core": 0.24052676558494568,
|
|
"loss/preference_sft": 0.2895175814628601,
|
|
"loss/reference_anchor": 0.18469442427158356,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.7074534893035889,
|
|
"rewards/margins": -0.06207428127527237,
|
|
"rewards/rejected": 1.769527792930603,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36278414726257324,
|
|
"drift/rejected_abs": 0.23082184791564941,
|
|
"epoch": 0.5829301797766231,
|
|
"grad_norm": 872.0,
|
|
"ht_mnpo/logit": 0.13208219408988953,
|
|
"ht_mnpo/residual": 0.1245821937918663,
|
|
"ht_mnpo/residual_abs": 0.31827038526535034,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.151452938229325e-08,
|
|
"logits/chosen": -2.1874964237213135,
|
|
"logits/rejected": -1.9628692865371704,
|
|
"logps/chosen": -0.2622188925743103,
|
|
"logps/rejected": -0.27765434980392456,
|
|
"loss": 1.3239902257919312,
|
|
"loss/core": 1.3052964210510254,
|
|
"loss/preference_sft": 0.2622188925743103,
|
|
"loss/reference_anchor": 0.9084656834602356,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6278419494628906,
|
|
"rewards/margins": 1.3208218812942505,
|
|
"rewards/rejected": 2.3070197105407715,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18900258839130402,
|
|
"drift/rejected_abs": 0.08098079264163971,
|
|
"epoch": 0.5877082960043003,
|
|
"grad_norm": 21.5,
|
|
"ht_mnpo/logit": 0.10846320539712906,
|
|
"ht_mnpo/residual": 0.10096319764852524,
|
|
"ht_mnpo/residual_abs": 0.11534751951694489,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.933376866888883e-08,
|
|
"logits/chosen": -2.2008748054504395,
|
|
"logits/rejected": -2.009446859359741,
|
|
"logps/chosen": -0.2769085466861725,
|
|
"logps/rejected": -0.29790812730789185,
|
|
"loss": 0.10706748068332672,
|
|
"loss/core": 0.1036563292145729,
|
|
"loss/preference_sft": 0.2769085466861725,
|
|
"loss/reference_anchor": 0.142866849899292,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8900257349014282,
|
|
"rewards/margins": 1.0846320390701294,
|
|
"rewards/rejected": 0.8053938746452332,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.238611102104187,
|
|
"drift/rejected_abs": 0.1267508566379547,
|
|
"epoch": 0.5924864122319775,
|
|
"grad_norm": 16.75,
|
|
"ht_mnpo/logit": 0.11258834600448608,
|
|
"ht_mnpo/residual": 0.10508836805820465,
|
|
"ht_mnpo/residual_abs": 0.15256913006305695,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.717394174850605e-08,
|
|
"logits/chosen": -2.3503708839416504,
|
|
"logits/rejected": -1.996098518371582,
|
|
"logps/chosen": -0.2766886353492737,
|
|
"logps/rejected": -0.2806140184402466,
|
|
"loss": 0.23508873581886292,
|
|
"loss/core": 0.23049023747444153,
|
|
"loss/preference_sft": 0.2766886353492737,
|
|
"loss/reference_anchor": 0.20225539803504944,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3849875926971436,
|
|
"rewards/margins": 1.1258835792541504,
|
|
"rewards/rejected": 1.2591038942337036,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2566293179988861,
|
|
"drift/rejected_abs": 0.17416846752166748,
|
|
"epoch": 0.5972645284596548,
|
|
"grad_norm": 1.4609375,
|
|
"ht_mnpo/logit": 0.08759935945272446,
|
|
"ht_mnpo/residual": 0.08009935915470123,
|
|
"ht_mnpo/residual_abs": 0.19110621511936188,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.503586084371926e-08,
|
|
"logits/chosen": -2.2542307376861572,
|
|
"logits/rejected": -1.9469505548477173,
|
|
"logps/chosen": -0.2917229235172272,
|
|
"logps/rejected": -0.30690133571624756,
|
|
"loss": 0.4792788028717041,
|
|
"loss/core": 0.46942344307899475,
|
|
"loss/preference_sft": 0.2917229235172272,
|
|
"loss/reference_anchor": 0.46359556913375854,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 2.564573287963867,
|
|
"rewards/margins": 0.8759937286376953,
|
|
"rewards/rejected": 1.688579797744751,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19692203402519226,
|
|
"drift/rejected_abs": 0.11281520128250122,
|
|
"epoch": 0.602042644687332,
|
|
"grad_norm": 12.9375,
|
|
"ht_mnpo/logit": 0.08999810367822647,
|
|
"ht_mnpo/residual": 0.08249811083078384,
|
|
"ht_mnpo/residual_abs": 0.10153649747371674,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.29203299993155e-08,
|
|
"logits/chosen": -2.2926273345947266,
|
|
"logits/rejected": -2.0748157501220703,
|
|
"logps/chosen": -0.2932100296020508,
|
|
"logps/rejected": -0.29019203782081604,
|
|
"loss": 0.0861930325627327,
|
|
"loss/core": 0.08310763537883759,
|
|
"loss/preference_sft": 0.2932100296020508,
|
|
"loss/reference_anchor": 0.1249486431479454,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.968793511390686,
|
|
"rewards/margins": 0.8999810218811035,
|
|
"rewards/rejected": 1.068812608718872,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18911968171596527,
|
|
"drift/rejected_abs": 0.11328919231891632,
|
|
"epoch": 0.6068207609150092,
|
|
"grad_norm": 80.0,
|
|
"ht_mnpo/logit": 0.07597161084413528,
|
|
"ht_mnpo/residual": 0.06847161799669266,
|
|
"ht_mnpo/residual_abs": 0.10563920438289642,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.082814477992656e-08,
|
|
"logits/chosen": -2.048794984817505,
|
|
"logits/rejected": -1.8921749591827393,
|
|
"logps/chosen": -0.2832937240600586,
|
|
"logps/rejected": -0.3034849762916565,
|
|
"loss": 0.1296667456626892,
|
|
"loss/core": 0.1261918693780899,
|
|
"loss/preference_sft": 0.2832937240600586,
|
|
"loss/reference_anchor": 0.14541494846343994,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.8911077976226807,
|
|
"rewards/margins": 0.7597162127494812,
|
|
"rewards/rejected": 1.1313915252685547,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21820612251758575,
|
|
"drift/rejected_abs": 0.14316417276859283,
|
|
"epoch": 0.6115988771426865,
|
|
"grad_norm": 1.2578125,
|
|
"ht_mnpo/logit": 0.07545202970504761,
|
|
"ht_mnpo/residual": 0.06795202940702438,
|
|
"ht_mnpo/residual_abs": 0.11145929247140884,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.87600919708494e-08,
|
|
"logits/chosen": -2.21930193901062,
|
|
"logits/rejected": -1.9665937423706055,
|
|
"logps/chosen": -0.26268836855888367,
|
|
"logps/rejected": -0.27242428064346313,
|
|
"loss": 0.09963403642177582,
|
|
"loss/core": 0.09529232233762741,
|
|
"loss/preference_sft": 0.26268836855888367,
|
|
"loss/reference_anchor": 0.19081640243530273,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.181434154510498,
|
|
"rewards/margins": 0.7545203566551208,
|
|
"rewards/rejected": 1.4269136190414429,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22275419533252716,
|
|
"drift/rejected_abs": 0.12799355387687683,
|
|
"epoch": 0.6163769933703638,
|
|
"grad_norm": 16.125,
|
|
"ht_mnpo/logit": 0.09602762758731842,
|
|
"ht_mnpo/residual": 0.08852764219045639,
|
|
"ht_mnpo/residual_abs": 0.19490960240364075,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.671694928216829e-08,
|
|
"logits/chosen": -2.2347233295440674,
|
|
"logits/rejected": -2.0241684913635254,
|
|
"logps/chosen": -0.27447259426116943,
|
|
"logps/rejected": -0.27523598074913025,
|
|
"loss": 0.44739675521850586,
|
|
"loss/core": 0.44131603837013245,
|
|
"loss/preference_sft": 0.27447259426116943,
|
|
"loss/reference_anchor": 0.2765862047672272,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.227006435394287,
|
|
"rewards/margins": 0.9602764248847961,
|
|
"rewards/rejected": 1.2667300701141357,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20366021990776062,
|
|
"drift/rejected_abs": 0.20054574310779572,
|
|
"epoch": 0.621155109598041,
|
|
"grad_norm": 278.0,
|
|
"ht_mnpo/logit": 0.003406074596568942,
|
|
"ht_mnpo/residual": -0.004093932919204235,
|
|
"ht_mnpo/residual_abs": 0.15771286189556122,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.469948505629e-08,
|
|
"logits/chosen": -2.2959370613098145,
|
|
"logits/rejected": -1.9112621545791626,
|
|
"logps/chosen": -0.286499559879303,
|
|
"logps/rejected": -0.30210891366004944,
|
|
"loss": 0.28895777463912964,
|
|
"loss/core": 0.2796488106250763,
|
|
"loss/preference_sft": 0.286499559879303,
|
|
"loss/reference_anchor": 0.4367947578430176,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.0365281105041504,
|
|
"rewards/margins": 0.0340607650578022,
|
|
"rewards/rejected": 2.002467632293701,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2379056215286255,
|
|
"drift/rejected_abs": 0.18813912570476532,
|
|
"epoch": 0.6259332258257182,
|
|
"grad_norm": 5.21875,
|
|
"ht_mnpo/logit": 0.05104045942425728,
|
|
"ht_mnpo/residual": 0.043540455400943756,
|
|
"ht_mnpo/residual_abs": 0.1262165606021881,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.270845797900168e-08,
|
|
"logits/chosen": -2.0275120735168457,
|
|
"logits/rejected": -1.7314637899398804,
|
|
"logps/chosen": -0.27975013852119446,
|
|
"logps/rejected": -0.2768488824367523,
|
|
"loss": 0.09684029966592789,
|
|
"loss/core": 0.08728532493114471,
|
|
"loss/preference_sft": 0.27975013852119446,
|
|
"loss/reference_anchor": 0.4497728943824768,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.378608226776123,
|
|
"rewards/margins": 0.5104045271873474,
|
|
"rewards/rejected": 1.8682035207748413,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12607762217521667,
|
|
"drift/rejected_abs": 0.08264373242855072,
|
|
"epoch": 0.6307113420533954,
|
|
"grad_norm": 16.5,
|
|
"ht_mnpo/logit": 0.043472785502672195,
|
|
"ht_mnpo/residual": 0.03597278520464897,
|
|
"ht_mnpo/residual_abs": 0.047445155680179596,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.0744616794160104e-08,
|
|
"logits/chosen": -2.3355751037597656,
|
|
"logits/rejected": -2.13651180267334,
|
|
"logps/chosen": -0.297650545835495,
|
|
"logps/rejected": -0.2983691096305847,
|
|
"loss": 0.014283724129199982,
|
|
"loss/core": 0.011677760630846024,
|
|
"loss/preference_sft": 0.297650545835495,
|
|
"loss/reference_anchor": 0.10053311288356781,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2598559856414795,
|
|
"rewards/margins": 0.43472784757614136,
|
|
"rewards/rejected": 0.8251279592514038,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12593212723731995,
|
|
"drift/rejected_abs": 0.09534545242786407,
|
|
"epoch": 0.6354894582810727,
|
|
"grad_norm": 0.12890625,
|
|
"ht_mnpo/logit": 0.030613893643021584,
|
|
"ht_mnpo/residual": 0.02311389520764351,
|
|
"ht_mnpo/residual_abs": 0.06221969053149223,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.880870002211963e-08,
|
|
"logits/chosen": -2.3577072620391846,
|
|
"logits/rejected": -2.1732094287872314,
|
|
"logps/chosen": -0.28674453496932983,
|
|
"logps/rejected": -0.28307873010635376,
|
|
"loss": 0.016248947009444237,
|
|
"loss/core": 0.014818708412349224,
|
|
"loss/preference_sft": 0.28674453496932983,
|
|
"loss/reference_anchor": 0.04283740743994713,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.2591238021850586,
|
|
"rewards/margins": 0.30613890290260315,
|
|
"rewards/rejected": 0.9529846906661987,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23004214465618134,
|
|
"drift/rejected_abs": 0.10869423300027847,
|
|
"epoch": 0.6402675745087499,
|
|
"grad_norm": 13.4375,
|
|
"ht_mnpo/logit": 0.12265989929437637,
|
|
"ht_mnpo/residual": 0.11515990644693375,
|
|
"ht_mnpo/residual_abs": 0.13466432690620422,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6901435682004996e-08,
|
|
"logits/chosen": -2.0747597217559814,
|
|
"logits/rejected": -1.8630584478378296,
|
|
"logps/chosen": -0.26450079679489136,
|
|
"logps/rejected": -0.28103771805763245,
|
|
"loss": 0.10082646459341049,
|
|
"loss/core": 0.09776544570922852,
|
|
"loss/preference_sft": 0.26450079679489136,
|
|
"loss/reference_anchor": 0.12660036981105804,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.30025315284729,
|
|
"rewards/margins": 1.2265989780426025,
|
|
"rewards/rejected": 1.0736541748046875,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16283902525901794,
|
|
"drift/rejected_abs": 0.1080724224448204,
|
|
"epoch": 0.6450456907364271,
|
|
"grad_norm": 17.375,
|
|
"ht_mnpo/logit": 0.0548507496714592,
|
|
"ht_mnpo/residual": 0.04735075309872627,
|
|
"ht_mnpo/residual_abs": 0.06609907001256943,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.502354101793314e-08,
|
|
"logits/chosen": -2.2743802070617676,
|
|
"logits/rejected": -2.0450034141540527,
|
|
"logps/chosen": -0.28120556473731995,
|
|
"logps/rejected": -0.28399839997291565,
|
|
"loss": 0.03949340060353279,
|
|
"loss/core": 0.036700859665870667,
|
|
"loss/preference_sft": 0.28120556473731995,
|
|
"loss/reference_anchor": 0.1115063801407814,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6273581981658936,
|
|
"rewards/margins": 0.548507571220398,
|
|
"rewards/rejected": 1.0788507461547852,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2523915767669678,
|
|
"drift/rejected_abs": 0.1427987664937973,
|
|
"epoch": 0.6498238069641044,
|
|
"grad_norm": 84.5,
|
|
"ht_mnpo/logit": 0.11365683376789093,
|
|
"ht_mnpo/residual": 0.1061568409204483,
|
|
"ht_mnpo/residual_abs": 0.22065427899360657,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3175722229287034e-08,
|
|
"logits/chosen": -2.2273154258728027,
|
|
"logits/rejected": -1.9926984310150146,
|
|
"logps/chosen": -0.30488717555999756,
|
|
"logps/rejected": -0.29885029792785645,
|
|
"loss": 0.5688546895980835,
|
|
"loss/core": 0.5607632994651794,
|
|
"loss/preference_sft": 0.30488717555999756,
|
|
"loss/reference_anchor": 0.3740817904472351,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 2.523131847381592,
|
|
"rewards/margins": 1.136568307876587,
|
|
"rewards/rejected": 1.386563777923584,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14107666909694672,
|
|
"drift/rejected_abs": 0.1379661113023758,
|
|
"epoch": 0.6546019231917817,
|
|
"grad_norm": 20.375,
|
|
"ht_mnpo/logit": 0.0031998082995414734,
|
|
"ht_mnpo/residual": -0.004300191067159176,
|
|
"ht_mnpo/residual_abs": 0.06062345951795578,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.135867420514276e-08,
|
|
"logits/chosen": -2.150909423828125,
|
|
"logits/rejected": -2.010291814804077,
|
|
"logps/chosen": -0.2966502010822296,
|
|
"logps/rejected": -0.2863856256008148,
|
|
"loss": 0.02938271500170231,
|
|
"loss/core": 0.02649505063891411,
|
|
"loss/preference_sft": 0.2966502010822296,
|
|
"loss/reference_anchor": 0.11471836268901825,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.4100998640060425,
|
|
"rewards/margins": 0.03199812024831772,
|
|
"rewards/rejected": 1.3781017065048218,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2356395274400711,
|
|
"drift/rejected_abs": 0.2509574294090271,
|
|
"epoch": 0.6593800394194589,
|
|
"grad_norm": 128.0,
|
|
"ht_mnpo/logit": -0.01525307446718216,
|
|
"ht_mnpo/residual": -0.02275308221578598,
|
|
"ht_mnpo/residual_abs": 0.23793688416481018,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.957308026295035e-08,
|
|
"logits/chosen": -2.145141839981079,
|
|
"logits/rejected": -1.8953478336334229,
|
|
"logps/chosen": -0.2728975713253021,
|
|
"logps/rejected": -0.2741245925426483,
|
|
"loss": 0.6547005772590637,
|
|
"loss/core": 0.6441726684570312,
|
|
"loss/preference_sft": 0.2728975713253021,
|
|
"loss/reference_anchor": 0.4991017282009125,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.3562231063842773,
|
|
"rewards/margins": -0.15253083407878876,
|
|
"rewards/rejected": 2.508754014968872,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23801188170909882,
|
|
"drift/rejected_abs": 0.13991716504096985,
|
|
"epoch": 0.6641581556471361,
|
|
"grad_norm": 2.9375,
|
|
"ht_mnpo/logit": 0.09799585491418839,
|
|
"ht_mnpo/residual": 0.09049585461616516,
|
|
"ht_mnpo/residual_abs": 0.12964896857738495,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7819611891566084e-08,
|
|
"logits/chosen": -2.3769142627716064,
|
|
"logits/rejected": -2.1696107387542725,
|
|
"logps/chosen": -0.28293538093566895,
|
|
"logps/rejected": -0.29280370473861694,
|
|
"loss": 0.3060581088066101,
|
|
"loss/core": 0.2948313355445862,
|
|
"loss/preference_sft": 0.28293538093566895,
|
|
"loss/reference_anchor": 0.5330466032028198,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.378937244415283,
|
|
"rewards/margins": 0.9799586534500122,
|
|
"rewards/rejected": 1.3989789485931396,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11122707277536392,
|
|
"drift/rejected_abs": 0.12570586800575256,
|
|
"epoch": 0.6689362718748133,
|
|
"grad_norm": 139.0,
|
|
"ht_mnpo/logit": -0.014081744477152824,
|
|
"ht_mnpo/residual": -0.021581750363111496,
|
|
"ht_mnpo/residual_abs": 0.08934501558542252,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.609892849873286e-08,
|
|
"logits/chosen": -2.561699867248535,
|
|
"logits/rejected": -2.2563445568084717,
|
|
"logps/chosen": -0.30130958557128906,
|
|
"logps/rejected": -0.31445008516311646,
|
|
"loss": 0.093419149518013,
|
|
"loss/core": 0.09064619243144989,
|
|
"loss/preference_sft": 0.30130958557128906,
|
|
"loss/reference_anchor": 0.10851649194955826,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.111412763595581,
|
|
"rewards/margins": -0.14081746339797974,
|
|
"rewards/rejected": 1.252230167388916,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.130570650100708,
|
|
"drift/rejected_abs": 0.1270247995853424,
|
|
"epoch": 0.6737143881024906,
|
|
"grad_norm": 55.0,
|
|
"ht_mnpo/logit": 0.003545860992744565,
|
|
"ht_mnpo/residual": -0.003954136278480291,
|
|
"ht_mnpo/residual_abs": 0.0973631739616394,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4411677163103894e-08,
|
|
"logits/chosen": -2.2780685424804688,
|
|
"logits/rejected": -2.014157772064209,
|
|
"logps/chosen": -0.29029467701911926,
|
|
"logps/rejected": -0.2877400517463684,
|
|
"loss": 0.07396967709064484,
|
|
"loss/core": 0.07192926108837128,
|
|
"loss/preference_sft": 0.29029467701911926,
|
|
"loss/reference_anchor": 0.07299144566059113,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 1.30570650100708,
|
|
"rewards/margins": 0.03545858711004257,
|
|
"rewards/rejected": 1.2702479362487793,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.0952615886926651,
|
|
"drift/rejected_abs": 0.047143109142780304,
|
|
"epoch": 0.6784925043301678,
|
|
"grad_norm": 139.0,
|
|
"ht_mnpo/logit": 0.04816994443535805,
|
|
"ht_mnpo/residual": 0.04066994786262512,
|
|
"ht_mnpo/residual_abs": 0.04842040687799454,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.2758492390902945e-08,
|
|
"logits/chosen": -2.2472968101501465,
|
|
"logits/rejected": -2.1181864738464355,
|
|
"logps/chosen": -0.2959393560886383,
|
|
"logps/rejected": -0.3067687153816223,
|
|
"loss": 0.015642011538147926,
|
|
"loss/core": 0.014619368128478527,
|
|
"loss/preference_sft": 0.2959393560886383,
|
|
"loss/reference_anchor": 0.021538157016038895,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.9488660097122192,
|
|
"rewards/margins": 0.48169946670532227,
|
|
"rewards/rejected": 0.46716657280921936,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1471184939146042,
|
|
"drift/rejected_abs": 0.19262142479419708,
|
|
"epoch": 0.683270620557845,
|
|
"grad_norm": 44.5,
|
|
"ht_mnpo/logit": -0.04537930339574814,
|
|
"ht_mnpo/residual": -0.052879296243190765,
|
|
"ht_mnpo/residual_abs": 0.1437372863292694,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.11399958773126e-08,
|
|
"logits/chosen": -2.2122180461883545,
|
|
"logits/rejected": -1.960108757019043,
|
|
"logps/chosen": -0.25510653853416443,
|
|
"logps/rejected": -0.2806130051612854,
|
|
"loss": 0.3858787417411804,
|
|
"loss/core": 0.3799152672290802,
|
|
"loss/preference_sft": 0.25510653853416443,
|
|
"loss/reference_anchor": 0.2726641297340393,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.470590353012085,
|
|
"rewards/margins": -0.45379310846328735,
|
|
"rewards/rejected": 1.924383521080017,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2561036944389343,
|
|
"drift/rejected_abs": 0.1267954558134079,
|
|
"epoch": 0.6880487367855223,
|
|
"grad_norm": 19.125,
|
|
"ht_mnpo/logit": 0.1294882744550705,
|
|
"ht_mnpo/residual": 0.12198827415704727,
|
|
"ht_mnpo/residual_abs": 0.17171990871429443,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9556796272679972e-08,
|
|
"logits/chosen": -2.2394776344299316,
|
|
"logits/rejected": -2.0022566318511963,
|
|
"logps/chosen": -0.30328303575515747,
|
|
"logps/rejected": -0.2942310869693756,
|
|
"loss": 0.3686460554599762,
|
|
"loss/core": 0.3613186478614807,
|
|
"loss/preference_sft": 0.30328303575515747,
|
|
"loss/reference_anchor": 0.3360404670238495,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.559324264526367,
|
|
"rewards/margins": 1.2948826551437378,
|
|
"rewards/rejected": 1.264441728591919,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20471441745758057,
|
|
"drift/rejected_abs": 0.12798653542995453,
|
|
"epoch": 0.6928268530131996,
|
|
"grad_norm": 11.3125,
|
|
"ht_mnpo/logit": 0.0769859105348587,
|
|
"ht_mnpo/residual": 0.06948591768741608,
|
|
"ht_mnpo/residual_abs": 0.10851018130779266,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8009488953628215e-08,
|
|
"logits/chosen": -2.1995251178741455,
|
|
"logits/rejected": -2.007638931274414,
|
|
"logps/chosen": -0.2995498478412628,
|
|
"logps/rejected": -0.29953497648239136,
|
|
"loss": 0.05993145704269409,
|
|
"loss/core": 0.056487929075956345,
|
|
"loss/preference_sft": 0.2995498478412628,
|
|
"loss/reference_anchor": 0.14222107827663422,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.044583797454834,
|
|
"rewards/margins": 0.7698591947555542,
|
|
"rewards/rejected": 1.2747247219085693,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3021154999732971,
|
|
"drift/rejected_abs": 0.28609389066696167,
|
|
"epoch": 0.6976049692408768,
|
|
"grad_norm": 7.25,
|
|
"ht_mnpo/logit": 0.022638099268078804,
|
|
"ht_mnpo/residual": 0.015138095244765282,
|
|
"ht_mnpo/residual_abs": 0.22438108921051025,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.649865579915976e-08,
|
|
"logits/chosen": -2.218268394470215,
|
|
"logits/rejected": -1.9819151163101196,
|
|
"logps/chosen": -0.2943797707557678,
|
|
"logps/rejected": -0.295765221118927,
|
|
"loss": 0.8258689641952515,
|
|
"loss/core": 0.8050146102905273,
|
|
"loss/preference_sft": 0.2943797707557678,
|
|
"loss/reference_anchor": 1.0132826566696167,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.0197532176971436,
|
|
"rewards/margins": 0.22638094425201416,
|
|
"rewards/rejected": 2.793372392654419,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22813507914543152,
|
|
"drift/rejected_abs": 0.1425250917673111,
|
|
"epoch": 0.702383085468554,
|
|
"grad_norm": 6.96875,
|
|
"ht_mnpo/logit": 0.08600185066461563,
|
|
"ht_mnpo/residual": 0.078501857817173,
|
|
"ht_mnpo/residual_abs": 0.12325754016637802,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5024864971835507e-08,
|
|
"logits/chosen": -2.2636451721191406,
|
|
"logits/rejected": -2.125495433807373,
|
|
"logps/chosen": -0.2841891646385193,
|
|
"logps/rejected": -0.29996174573898315,
|
|
"loss": 0.22820651531219482,
|
|
"loss/core": 0.2230566293001175,
|
|
"loss/preference_sft": 0.2841891646385193,
|
|
"loss/reference_anchor": 0.22907507419586182,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.281350612640381,
|
|
"rewards/margins": 0.8600185513496399,
|
|
"rewards/rejected": 1.4213320016860962,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2448999136686325,
|
|
"drift/rejected_abs": 0.28253763914108276,
|
|
"epoch": 0.7071612016962313,
|
|
"grad_norm": 792.0,
|
|
"ht_mnpo/logit": -0.0376126654446125,
|
|
"ht_mnpo/residual": -0.045112669467926025,
|
|
"ht_mnpo/residual_abs": 0.2788679897785187,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3588670704111997e-08,
|
|
"logits/chosen": -2.1962971687316895,
|
|
"logits/rejected": -1.9129769802093506,
|
|
"logps/chosen": -0.30221128463745117,
|
|
"logps/rejected": -0.31767523288726807,
|
|
"loss": 0.8641053438186646,
|
|
"loss/core": 0.8473947644233704,
|
|
"loss/preference_sft": 0.30221128463745117,
|
|
"loss/reference_anchor": 0.8053110837936401,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.4467480182647705,
|
|
"rewards/margins": -0.37612658739089966,
|
|
"rewards/rejected": 2.8228747844696045,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14628183841705322,
|
|
"drift/rejected_abs": 0.19587254524230957,
|
|
"epoch": 0.7119393179239085,
|
|
"grad_norm": 168.0,
|
|
"ht_mnpo/logit": -0.04983377456665039,
|
|
"ht_mnpo/residual": -0.05733378604054451,
|
|
"ht_mnpo/residual_abs": 0.15143811702728271,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.219061308991721e-08,
|
|
"logits/chosen": -2.247026205062866,
|
|
"logits/rejected": -1.9038165807724,
|
|
"logps/chosen": -0.284883588552475,
|
|
"logps/rejected": -0.2793378233909607,
|
|
"loss": 0.5056548714637756,
|
|
"loss/core": 0.49778372049331665,
|
|
"loss/preference_sft": 0.284883588552475,
|
|
"loss/reference_anchor": 0.365068644285202,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 1.4588829278945923,
|
|
"rewards/margins": -0.49833768606185913,
|
|
"rewards/rejected": 1.9572206735610962,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14641818404197693,
|
|
"drift/rejected_abs": 0.16142067313194275,
|
|
"epoch": 0.7167174341515857,
|
|
"grad_norm": 580.0,
|
|
"ht_mnpo/logit": -0.014964533038437366,
|
|
"ht_mnpo/residual": -0.022464536130428314,
|
|
"ht_mnpo/residual_abs": 0.1398034393787384,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0831217881543557e-08,
|
|
"logits/chosen": -2.2321152687072754,
|
|
"logits/rejected": -2.0529208183288574,
|
|
"logps/chosen": -0.30759716033935547,
|
|
"logps/rejected": -0.36361774802207947,
|
|
"loss": 0.17265477776527405,
|
|
"loss/core": 0.1684485375881195,
|
|
"loss/preference_sft": 0.30759716033935547,
|
|
"loss/reference_anchor": 0.17955359816551208,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.4617996215820312,
|
|
"rewards/margins": -0.14964532852172852,
|
|
"rewards/rejected": 1.6114448308944702,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2566590905189514,
|
|
"drift/rejected_abs": 0.1291174292564392,
|
|
"epoch": 0.7214955503792629,
|
|
"grad_norm": 0.240234375,
|
|
"ht_mnpo/logit": 0.15318958461284637,
|
|
"ht_mnpo/residual": 0.14568959176540375,
|
|
"ht_mnpo/residual_abs": 0.17443707585334778,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.951099629193366e-08,
|
|
"logits/chosen": -2.130836009979248,
|
|
"logits/rejected": -1.8832871913909912,
|
|
"logps/chosen": -0.2614069879055023,
|
|
"logps/rejected": -0.29673653841018677,
|
|
"loss": 0.428077757358551,
|
|
"loss/core": 0.42090481519699097,
|
|
"loss/preference_sft": 0.2614069879055023,
|
|
"loss/reference_anchor": 0.33250653743743896,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.5654702186584473,
|
|
"rewards/margins": 1.5318958759307861,
|
|
"rewards/rejected": 1.0335744619369507,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13500568270683289,
|
|
"drift/rejected_abs": 0.1280622035264969,
|
|
"epoch": 0.7262736666069403,
|
|
"grad_norm": 20.0,
|
|
"ht_mnpo/logit": 0.006745192222297192,
|
|
"ht_mnpo/residual": -0.0007548049325123429,
|
|
"ht_mnpo/residual_abs": 0.08610773086547852,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.823044480243431e-08,
|
|
"logits/chosen": -2.1745476722717285,
|
|
"logits/rejected": -1.9323323965072632,
|
|
"logps/chosen": -0.27481788396835327,
|
|
"logps/rejected": -0.28048986196517944,
|
|
"loss": 0.07846397906541824,
|
|
"loss/core": 0.07617657631635666,
|
|
"loss/preference_sft": 0.27481788396835327,
|
|
"loss/reference_anchor": 0.08688807487487793,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 1.347092628479004,
|
|
"rewards/margins": 0.06745192408561707,
|
|
"rewards/rejected": 1.2796406745910645,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2502586841583252,
|
|
"drift/rejected_abs": 0.11848635971546173,
|
|
"epoch": 0.7310517828346175,
|
|
"grad_norm": 394.0,
|
|
"ht_mnpo/logit": 0.13317808508872986,
|
|
"ht_mnpo/residual": 0.12567809224128723,
|
|
"ht_mnpo/residual_abs": 0.16746442019939423,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.699004497608994e-08,
|
|
"logits/chosen": -2.1543080806732178,
|
|
"logits/rejected": -1.9849952459335327,
|
|
"logps/chosen": -0.2812741994857788,
|
|
"logps/rejected": -0.2872457504272461,
|
|
"loss": 0.33080416917800903,
|
|
"loss/core": 0.32540163397789,
|
|
"loss/preference_sft": 0.2812741994857788,
|
|
"loss/reference_anchor": 0.24199938774108887,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.501790761947632,
|
|
"rewards/margins": 1.3317807912826538,
|
|
"rewards/rejected": 1.1700100898742676,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16793134808540344,
|
|
"drift/rejected_abs": 0.12145377695560455,
|
|
"epoch": 0.7358298990622947,
|
|
"grad_norm": 13.25,
|
|
"ht_mnpo/logit": 0.046602535992860794,
|
|
"ht_mnpo/residual": 0.03910253569483757,
|
|
"ht_mnpo/residual_abs": 0.08376048505306244,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5790263276546658e-08,
|
|
"logits/chosen": -2.3451311588287354,
|
|
"logits/rejected": -2.0735201835632324,
|
|
"logps/chosen": -0.2896055281162262,
|
|
"logps/rejected": -0.2964259386062622,
|
|
"loss": 0.03520764783024788,
|
|
"loss/core": 0.032552026212215424,
|
|
"loss/preference_sft": 0.2896055281162262,
|
|
"loss/reference_anchor": 0.10382010042667389,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.6792783737182617,
|
|
"rewards/margins": 0.4660254120826721,
|
|
"rewards/rejected": 1.2132529020309448,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27133306860923767,
|
|
"drift/rejected_abs": 0.18516871333122253,
|
|
"epoch": 0.7406080152899719,
|
|
"grad_norm": 7.71875,
|
|
"ht_mnpo/logit": 0.08782286942005157,
|
|
"ht_mnpo/residual": 0.08032286167144775,
|
|
"ht_mnpo/residual_abs": 0.19639639556407928,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.4631550892634126e-08,
|
|
"logits/chosen": -2.123995304107666,
|
|
"logits/rejected": -1.9403555393218994,
|
|
"logps/chosen": -0.25651782751083374,
|
|
"logps/rejected": -0.27262482047080994,
|
|
"loss": 0.2998473048210144,
|
|
"loss/core": 0.29379308223724365,
|
|
"loss/preference_sft": 0.25651782751083374,
|
|
"loss/reference_anchor": 0.2770591974258423,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.711564540863037,
|
|
"rewards/margins": 0.8782287836074829,
|
|
"rewards/rejected": 1.8333358764648438,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2392168492078781,
|
|
"drift/rejected_abs": 0.08553509414196014,
|
|
"epoch": 0.7453861315176492,
|
|
"grad_norm": 396.0,
|
|
"ht_mnpo/logit": 0.15414533019065857,
|
|
"ht_mnpo/residual": 0.14664529263973236,
|
|
"ht_mnpo/residual_abs": 0.15226368606090546,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3514343568692132e-08,
|
|
"logits/chosen": -2.282392740249634,
|
|
"logits/rejected": -2.234204053878784,
|
|
"logps/chosen": -0.2903969883918762,
|
|
"logps/rejected": -0.29160791635513306,
|
|
"loss": 0.4157602787017822,
|
|
"loss/core": 0.4093073308467865,
|
|
"loss/preference_sft": 0.2903969883918762,
|
|
"loss/reference_anchor": 0.29360833764076233,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3913302421569824,
|
|
"rewards/margins": 1.5414531230926514,
|
|
"rewards/rejected": 0.8498773574829102,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2589685916900635,
|
|
"drift/rejected_abs": 0.1315867006778717,
|
|
"epoch": 0.7501642477453264,
|
|
"grad_norm": 4.5,
|
|
"ht_mnpo/logit": 0.127206951379776,
|
|
"ht_mnpo/residual": 0.11970694363117218,
|
|
"ht_mnpo/residual_abs": 0.14667531847953796,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2439061440704724e-08,
|
|
"logits/chosen": -1.9296722412109375,
|
|
"logits/rejected": -1.7011972665786743,
|
|
"logps/chosen": -0.3024870455265045,
|
|
"logps/rejected": -0.3112749755382538,
|
|
"loss": 0.255341112613678,
|
|
"loss/core": 0.24924786388874054,
|
|
"loss/preference_sft": 0.3024870455265045,
|
|
"loss/reference_anchor": 0.2744150459766388,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.585298538208008,
|
|
"rewards/margins": 1.2720695734024048,
|
|
"rewards/rejected": 1.3132292032241821,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18013639748096466,
|
|
"drift/rejected_abs": 0.11114680767059326,
|
|
"epoch": 0.7549423639730036,
|
|
"grad_norm": 30.75,
|
|
"ht_mnpo/logit": 0.06904369592666626,
|
|
"ht_mnpo/residual": 0.06154369190335274,
|
|
"ht_mnpo/residual_abs": 0.11316666752099991,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1406108878304468e-08,
|
|
"logits/chosen": -2.1777164936065674,
|
|
"logits/rejected": -1.9580738544464111,
|
|
"logps/chosen": -0.32455024123191833,
|
|
"logps/rejected": -0.33517375588417053,
|
|
"loss": 0.3174150884151459,
|
|
"loss/core": 0.3100249171257019,
|
|
"loss/preference_sft": 0.32455024123191833,
|
|
"loss/reference_anchor": 0.3370537757873535,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.800376534461975,
|
|
"rewards/margins": 0.6904369592666626,
|
|
"rewards/rejected": 1.1099395751953125,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3415032923221588,
|
|
"drift/rejected_abs": 0.18802081048488617,
|
|
"epoch": 0.7597204802006808,
|
|
"grad_norm": 9.125,
|
|
"ht_mnpo/logit": 0.1534634530544281,
|
|
"ht_mnpo/residual": 0.14596346020698547,
|
|
"ht_mnpo/residual_abs": 0.247287318110466,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0415874332705381e-08,
|
|
"logits/chosen": -2.1014323234558105,
|
|
"logits/rejected": -1.8502187728881836,
|
|
"logps/chosen": -0.33924803137779236,
|
|
"logps/rejected": -0.32133159041404724,
|
|
"loss": 0.8026763796806335,
|
|
"loss/core": 0.7806569337844849,
|
|
"loss/preference_sft": 0.33924803137779236,
|
|
"loss/reference_anchor": 1.067044973373413,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 3.411249876022339,
|
|
"rewards/margins": 1.5346347093582153,
|
|
"rewards/rejected": 1.8766151666641235,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11637525260448456,
|
|
"drift/rejected_abs": 0.07857302576303482,
|
|
"epoch": 0.7644985964283582,
|
|
"grad_norm": 1.5859375,
|
|
"ht_mnpo/logit": 0.03819254785776138,
|
|
"ht_mnpo/residual": 0.030692553147673607,
|
|
"ht_mnpo/residual_abs": 0.04601796343922615,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.468730190622484e-09,
|
|
"logits/chosen": -2.430523633956909,
|
|
"logits/rejected": -2.217977523803711,
|
|
"logps/chosen": -0.2681643068790436,
|
|
"logps/rejected": -0.2834469676017761,
|
|
"loss": 0.013186362572014332,
|
|
"loss/core": 0.01157904788851738,
|
|
"loss/preference_sft": 0.2681643068790436,
|
|
"loss/reference_anchor": 0.05354924127459526,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.1637526750564575,
|
|
"rewards/margins": 0.3819255232810974,
|
|
"rewards/rejected": 0.7818271517753601,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18257346749305725,
|
|
"drift/rejected_abs": 0.17861232161521912,
|
|
"epoch": 0.7692767126560354,
|
|
"grad_norm": 36.5,
|
|
"ht_mnpo/logit": 0.004233963787555695,
|
|
"ht_mnpo/residual": -0.0032660351134836674,
|
|
"ht_mnpo/residual_abs": 0.1263841688632965,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.565032634232194e-09,
|
|
"logits/chosen": -2.478210210800171,
|
|
"logits/rejected": -2.1900885105133057,
|
|
"logps/chosen": -0.28286072611808777,
|
|
"logps/rejected": -0.30062970519065857,
|
|
"loss": 0.15072055160999298,
|
|
"loss/core": 0.14563368260860443,
|
|
"loss/preference_sft": 0.28286072611808777,
|
|
"loss/reference_anchor": 0.22605614364147186,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8255739212036133,
|
|
"rewards/margins": 0.0423397533595562,
|
|
"rewards/rejected": 1.7832343578338623,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1565002202987671,
|
|
"drift/rejected_abs": 0.13715356588363647,
|
|
"epoch": 0.7740548288837126,
|
|
"grad_norm": 260.0,
|
|
"ht_mnpo/logit": 0.02574211359024048,
|
|
"ht_mnpo/residual": 0.018242118880152702,
|
|
"ht_mnpo/residual_abs": 0.10176096856594086,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.70512150722702e-09,
|
|
"logits/chosen": -2.2540268898010254,
|
|
"logits/rejected": -1.9384829998016357,
|
|
"logps/chosen": -0.26865464448928833,
|
|
"logps/rejected": -0.279658704996109,
|
|
"loss": 0.06719930469989777,
|
|
"loss/core": 0.06475383788347244,
|
|
"loss/preference_sft": 0.26865464448928833,
|
|
"loss/reference_anchor": 0.0954083725810051,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.5635719299316406,
|
|
"rewards/margins": 0.2574211657047272,
|
|
"rewards/rejected": 1.3061506748199463,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2879694104194641,
|
|
"drift/rejected_abs": 0.14862984418869019,
|
|
"epoch": 0.7788329451113898,
|
|
"grad_norm": 516.0,
|
|
"ht_mnpo/logit": 0.14298096299171448,
|
|
"ht_mnpo/residual": 0.13548094034194946,
|
|
"ht_mnpo/residual_abs": 0.20370686054229736,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.8893201870139915e-09,
|
|
"logits/chosen": -2.174163341522217,
|
|
"logits/rejected": -1.8267030715942383,
|
|
"logps/chosen": -0.27633410692214966,
|
|
"logps/rejected": -0.29714980721473694,
|
|
"loss": 0.497852623462677,
|
|
"loss/core": 0.4906894266605377,
|
|
"loss/preference_sft": 0.27633410692214966,
|
|
"loss/reference_anchor": 0.33052676916122437,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 2.8788421154022217,
|
|
"rewards/margins": 1.4298094511032104,
|
|
"rewards/rejected": 1.4490330219268799,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3162073791027069,
|
|
"drift/rejected_abs": 0.1260301172733307,
|
|
"epoch": 0.7836110613390671,
|
|
"grad_norm": 844.0,
|
|
"ht_mnpo/logit": 0.19014951586723328,
|
|
"ht_mnpo/residual": 0.18264950811862946,
|
|
"ht_mnpo/residual_abs": 0.18700771033763885,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.117935463105808e-09,
|
|
"logits/chosen": -2.2044544219970703,
|
|
"logits/rejected": -2.0138018131256104,
|
|
"logps/chosen": -0.26344865560531616,
|
|
"logps/rejected": -0.2816709578037262,
|
|
"loss": 0.5444236993789673,
|
|
"loss/core": 0.535254716873169,
|
|
"loss/preference_sft": 0.26344865560531616,
|
|
"loss/reference_anchor": 0.4321085810661316,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.161675453186035,
|
|
"rewards/margins": 1.9014952182769775,
|
|
"rewards/rejected": 1.260180115699768,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17280279099941254,
|
|
"drift/rejected_abs": 0.11558995395898819,
|
|
"epoch": 0.7883891775667443,
|
|
"grad_norm": 61.75,
|
|
"ht_mnpo/logit": 0.05732642859220505,
|
|
"ht_mnpo/residual": 0.04982643201947212,
|
|
"ht_mnpo/residual_abs": 0.11239101737737656,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.391257421749826e-09,
|
|
"logits/chosen": -2.185276508331299,
|
|
"logits/rejected": -1.9858713150024414,
|
|
"logps/chosen": -0.27167096734046936,
|
|
"logps/rejected": -0.2912501096725464,
|
|
"loss": 0.10265527665615082,
|
|
"loss/core": 0.09941322356462479,
|
|
"loss/preference_sft": 0.27167096734046936,
|
|
"loss/reference_anchor": 0.13493572175502777,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.72737717628479,
|
|
"rewards/margins": 0.5732643008232117,
|
|
"rewards/rejected": 1.1541129350662231,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3662811517715454,
|
|
"drift/rejected_abs": 0.1733158379793167,
|
|
"epoch": 0.7931672937944215,
|
|
"grad_norm": 240.0,
|
|
"ht_mnpo/logit": 0.1925339549779892,
|
|
"ht_mnpo/residual": 0.18503394722938538,
|
|
"ht_mnpo/residual_abs": 0.2609553933143616,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.709559336838462e-09,
|
|
"logits/chosen": -2.038930654525757,
|
|
"logits/rejected": -1.8778717517852783,
|
|
"logps/chosen": -0.3013201355934143,
|
|
"logps/rejected": -0.30770185589790344,
|
|
"loss": 0.6489893198013306,
|
|
"loss/core": 0.6383043527603149,
|
|
"loss/preference_sft": 0.3013201355934143,
|
|
"loss/reference_anchor": 0.5041123032569885,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 3.658099412918091,
|
|
"rewards/margins": 1.9253393411636353,
|
|
"rewards/rejected": 1.7327598333358765,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21075081825256348,
|
|
"drift/rejected_abs": 0.11554797738790512,
|
|
"epoch": 0.7979454100220987,
|
|
"grad_norm": 7.4375,
|
|
"ht_mnpo/logit": 0.09528296440839767,
|
|
"ht_mnpo/residual": 0.08778297901153564,
|
|
"ht_mnpo/residual_abs": 0.1566668152809143,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.073097567142025e-09,
|
|
"logits/chosen": -2.253007173538208,
|
|
"logits/rejected": -1.9240471124649048,
|
|
"logps/chosen": -0.2810296416282654,
|
|
"logps/rejected": -0.288202702999115,
|
|
"loss": 0.2865773141384125,
|
|
"loss/core": 0.2817952632904053,
|
|
"loss/preference_sft": 0.2810296416282654,
|
|
"loss/reference_anchor": 0.21100135147571564,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.1059257984161377,
|
|
"rewards/margins": 0.9528296589851379,
|
|
"rewards/rejected": 1.1530958414077759,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16927887499332428,
|
|
"drift/rejected_abs": 0.13110263645648956,
|
|
"epoch": 0.802723526249776,
|
|
"grad_norm": 27.875,
|
|
"ht_mnpo/logit": 0.038380250334739685,
|
|
"ht_mnpo/residual": 0.030880261212587357,
|
|
"ht_mnpo/residual_abs": 0.07921403646469116,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.482111459902695e-09,
|
|
"logits/chosen": -2.3635165691375732,
|
|
"logits/rejected": -2.133021831512451,
|
|
"logps/chosen": -0.27768105268478394,
|
|
"logps/rejected": -0.2969363331794739,
|
|
"loss": 0.031025763601064682,
|
|
"loss/core": 0.02740824781358242,
|
|
"loss/preference_sft": 0.27768105268478394,
|
|
"loss/reference_anchor": 0.15310758352279663,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.6914350986480713,
|
|
"rewards/margins": 0.383802592754364,
|
|
"rewards/rejected": 1.307632565498352,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23839470744132996,
|
|
"drift/rejected_abs": 0.10520993173122406,
|
|
"epoch": 0.8075016424774533,
|
|
"grad_norm": 520.0,
|
|
"ht_mnpo/logit": 0.13712438941001892,
|
|
"ht_mnpo/residual": 0.1296243816614151,
|
|
"ht_mnpo/residual_abs": 0.17075686156749725,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9368232608258797e-09,
|
|
"logits/chosen": -2.2205090522766113,
|
|
"logits/rejected": -1.973059058189392,
|
|
"logps/chosen": -0.27566519379615784,
|
|
"logps/rejected": -0.27959397435188293,
|
|
"loss": 0.3910796046257019,
|
|
"loss/core": 0.38557177782058716,
|
|
"loss/preference_sft": 0.27566519379615784,
|
|
"loss/reference_anchor": 0.24782493710517883,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3833117485046387,
|
|
"rewards/margins": 1.3712438344955444,
|
|
"rewards/rejected": 1.0120675563812256,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2898247539997101,
|
|
"drift/rejected_abs": 0.20496828854084015,
|
|
"epoch": 0.8122797587051305,
|
|
"grad_norm": 30.25,
|
|
"ht_mnpo/logit": 0.08481884747743607,
|
|
"ht_mnpo/residual": 0.07731884717941284,
|
|
"ht_mnpo/residual_abs": 0.16502617299556732,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4374380305025866e-09,
|
|
"logits/chosen": -2.1580593585968018,
|
|
"logits/rejected": -2.01873779296875,
|
|
"logps/chosen": -0.28473132848739624,
|
|
"logps/rejected": -0.30126944184303284,
|
|
"loss": 0.3230627179145813,
|
|
"loss/core": 0.3124871850013733,
|
|
"loss/preference_sft": 0.28473132848739624,
|
|
"loss/reference_anchor": 0.500304639339447,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.89532732963562,
|
|
"rewards/margins": 0.8481884002685547,
|
|
"rewards/rejected": 2.0471386909484863,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2713679075241089,
|
|
"drift/rejected_abs": 0.1950051337480545,
|
|
"epoch": 0.8170578749328078,
|
|
"grad_norm": 143.0,
|
|
"ht_mnpo/logit": 0.08360113203525543,
|
|
"ht_mnpo/residual": 0.07610112428665161,
|
|
"ht_mnpo/residual_abs": 0.24734120070934296,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.984143567294594e-09,
|
|
"logits/chosen": -2.0692951679229736,
|
|
"logits/rejected": -1.8935229778289795,
|
|
"logps/chosen": -0.2885996401309967,
|
|
"logps/rejected": -0.2952654957771301,
|
|
"loss": 0.7806990742683411,
|
|
"loss/core": 0.7664183974266052,
|
|
"loss/preference_sft": 0.2885996401309967,
|
|
"loss/reference_anchor": 0.6851747632026672,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.7128584384918213,
|
|
"rewards/margins": 0.8360112309455872,
|
|
"rewards/rejected": 1.876847267150879,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17620348930358887,
|
|
"drift/rejected_abs": 0.16157342493534088,
|
|
"epoch": 0.821835991160485,
|
|
"grad_norm": 278.0,
|
|
"ht_mnpo/logit": 0.01473709475249052,
|
|
"ht_mnpo/residual": 0.007237098179757595,
|
|
"ht_mnpo/residual_abs": 0.1026911735534668,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.577110336711096e-09,
|
|
"logits/chosen": -2.147317409515381,
|
|
"logits/rejected": -1.9148515462875366,
|
|
"logps/chosen": -0.29422527551651,
|
|
"logps/rejected": -0.30900320410728455,
|
|
"loss": 0.11610148102045059,
|
|
"loss/core": 0.11178989708423615,
|
|
"loss/preference_sft": 0.29422527551651,
|
|
"loss/reference_anchor": 0.18615707755088806,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.7592500448226929,
|
|
"rewards/margins": 0.14737097918987274,
|
|
"rewards/rejected": 1.6118791103363037,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1526646763086319,
|
|
"drift/rejected_abs": 0.18857529759407043,
|
|
"epoch": 0.8266141073881622,
|
|
"grad_norm": 36.25,
|
|
"ht_mnpo/logit": -0.035790909081697464,
|
|
"ht_mnpo/residual": -0.04329090565443039,
|
|
"ht_mnpo/residual_abs": 0.15114633738994598,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2164914073037048e-09,
|
|
"logits/chosen": -2.1270697116851807,
|
|
"logits/rejected": -1.9014345407485962,
|
|
"logps/chosen": -0.2999208867549896,
|
|
"logps/rejected": -0.30866190791130066,
|
|
"loss": 0.393560528755188,
|
|
"loss/core": 0.3873744606971741,
|
|
"loss/preference_sft": 0.2999208867549896,
|
|
"loss/reference_anchor": 0.27931293845176697,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 1.5241281986236572,
|
|
"rewards/margins": -0.3579089939594269,
|
|
"rewards/rejected": 1.8820374011993408,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29999783635139465,
|
|
"drift/rejected_abs": 0.1245269924402237,
|
|
"epoch": 0.8313922236158394,
|
|
"grad_norm": 2.25,
|
|
"ht_mnpo/logit": 0.1785384863615036,
|
|
"ht_mnpo/residual": 0.17103847861289978,
|
|
"ht_mnpo/residual_abs": 0.18839120864868164,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.024223931035357e-10,
|
|
"logits/chosen": -2.1790618896484375,
|
|
"logits/rejected": -2.021745204925537,
|
|
"logps/chosen": -0.2831207811832428,
|
|
"logps/rejected": -0.3100909888744354,
|
|
"loss": 0.4790888726711273,
|
|
"loss/core": 0.4709659218788147,
|
|
"loss/preference_sft": 0.2831207811832428,
|
|
"loss/reference_anchor": 0.37783652544021606,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9999780654907227,
|
|
"rewards/margins": 1.7853851318359375,
|
|
"rewards/rejected": 1.2145931720733643,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13325706124305725,
|
|
"drift/rejected_abs": 0.06335233151912689,
|
|
"epoch": 0.8361703398435167,
|
|
"grad_norm": 22.125,
|
|
"ht_mnpo/logit": 0.06985118985176086,
|
|
"ht_mnpo/residual": 0.06235118955373764,
|
|
"ht_mnpo/residual_abs": 0.08558257669210434,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.350214026223516e-10,
|
|
"logits/chosen": -2.2370076179504395,
|
|
"logits/rejected": -2.0332276821136475,
|
|
"logps/chosen": -0.31737005710601807,
|
|
"logps/rejected": -0.32498207688331604,
|
|
"loss": 0.06058365851640701,
|
|
"loss/core": 0.058684103190898895,
|
|
"loss/preference_sft": 0.31737005710601807,
|
|
"loss/reference_anchor": 0.06324069201946259,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.3312170505523682,
|
|
"rewards/margins": 0.6985118985176086,
|
|
"rewards/rejected": 0.63270503282547,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.376343309879303,
|
|
"drift/rejected_abs": 0.2587401270866394,
|
|
"epoch": 0.840948456071194,
|
|
"grad_norm": 65.0,
|
|
"ht_mnpo/logit": 0.11799073219299316,
|
|
"ht_mnpo/residual": 0.11049073934555054,
|
|
"ht_mnpo/residual_abs": 0.21407637000083923,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.143889944367429e-10,
|
|
"logits/chosen": -1.9316622018814087,
|
|
"logits/rejected": -1.8352222442626953,
|
|
"logps/chosen": -0.29632678627967834,
|
|
"logps/rejected": -0.26413092017173767,
|
|
"loss": 0.603792667388916,
|
|
"loss/core": 0.5875565409660339,
|
|
"loss/preference_sft": 0.29632678627967834,
|
|
"loss/reference_anchor": 0.7821727991104126,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 3.7634329795837402,
|
|
"rewards/margins": 1.179907202720642,
|
|
"rewards/rejected": 2.5835256576538086,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2143622189760208,
|
|
"drift/rejected_abs": 0.10691346973180771,
|
|
"epoch": 0.8457265722988712,
|
|
"grad_norm": 1.1640625,
|
|
"ht_mnpo/logit": 0.10745616257190704,
|
|
"ht_mnpo/residual": 0.09995613992214203,
|
|
"ht_mnpo/residual_abs": 0.1442701518535614,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.406081393722531e-10,
|
|
"logits/chosen": -2.3482344150543213,
|
|
"logits/rejected": -2.0240073204040527,
|
|
"logps/chosen": -0.3080643117427826,
|
|
"logps/rejected": -0.3035167157649994,
|
|
"loss": 0.7237991094589233,
|
|
"loss/core": 0.7135142087936401,
|
|
"loss/preference_sft": 0.3080643117427826,
|
|
"loss/reference_anchor": 0.4834347665309906,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.1424150466918945,
|
|
"rewards/margins": 1.074561595916748,
|
|
"rewards/rejected": 1.067853331565857,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29515132308006287,
|
|
"drift/rejected_abs": 0.269777774810791,
|
|
"epoch": 0.8505046885265484,
|
|
"grad_norm": 9.25,
|
|
"ht_mnpo/logit": 0.025372039526700974,
|
|
"ht_mnpo/residual": 0.017872031778097153,
|
|
"ht_mnpo/residual_abs": 0.2621416449546814,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1374418930135133e-10,
|
|
"logits/chosen": -2.138016939163208,
|
|
"logits/rejected": -1.8397918939590454,
|
|
"logps/chosen": -0.2903136909008026,
|
|
"logps/rejected": -0.3230307400226593,
|
|
"loss": 1.0775001049041748,
|
|
"loss/core": 1.0594369173049927,
|
|
"loss/preference_sft": 0.2903136909008026,
|
|
"loss/reference_anchor": 0.8741267919540405,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.951321840286255,
|
|
"rewards/margins": 0.25372037291526794,
|
|
"rewards/rejected": 2.697601795196533,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1901109516620636,
|
|
"drift/rejected_abs": 0.11028845608234406,
|
|
"epoch": 0.8552828047542257,
|
|
"grad_norm": 7.53125,
|
|
"ht_mnpo/logit": 0.07955867052078247,
|
|
"ht_mnpo/residual": 0.07205866277217865,
|
|
"ht_mnpo/residual_abs": 0.14294487237930298,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3844852567285756e-11,
|
|
"logits/chosen": -2.2701876163482666,
|
|
"logits/rejected": -2.0042386054992676,
|
|
"logps/chosen": -0.2893333435058594,
|
|
"logps/rejected": -0.2989356517791748,
|
|
"loss": 0.26186782121658325,
|
|
"loss/core": 0.2570946514606476,
|
|
"loss/preference_sft": 0.2893333435058594,
|
|
"loss/reference_anchor": 0.20972399413585663,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.8983943462371826,
|
|
"rewards/margins": 0.7955867052078247,
|
|
"rewards/rejected": 1.1028077602386475,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14426791667938232,
|
|
"drift/rejected_abs": 0.09550980478525162,
|
|
"epoch": 0.8600609209819029,
|
|
"grad_norm": 1.375,
|
|
"ht_mnpo/logit": 0.04878208041191101,
|
|
"ht_mnpo/residual": 0.04128208011388779,
|
|
"ht_mnpo/residual_abs": 0.09187668561935425,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.401760429905703e-13,
|
|
"logits/chosen": -2.105118751525879,
|
|
"logits/rejected": -1.8005974292755127,
|
|
"logps/chosen": -0.2834312915802002,
|
|
"logps/rejected": -0.30577853322029114,
|
|
"loss": 0.10151888430118561,
|
|
"loss/core": 0.09912057220935822,
|
|
"loss/preference_sft": 0.2834312915802002,
|
|
"loss/reference_anchor": 0.09157194197177887,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.4408460855484009,
|
|
"rewards/margins": 0.4878207743167877,
|
|
"rewards/rejected": 0.953025221824646,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8600609209819029,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.3323093075222439,
|
|
"train_runtime": 7030.1668,
|
|
"train_samples_per_second": 2.048,
|
|
"train_steps_per_second": 0.128
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|