3824 lines
143 KiB
JSON
3824 lines
143 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8589835361488904,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004772130756382725,
|
|
"grad_norm": 7.90625,
|
|
"ht_mnpo/logit": 0.12129320949316025,
|
|
"ht_mnpo/residual": 0.11379321664571762,
|
|
"ht_mnpo/residual_abs": 0.14974038302898407,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -2.319894313812256,
|
|
"logits/rejected": -2.253976345062256,
|
|
"logps/chosen": -0.2934279143810272,
|
|
"logps/rejected": -0.2908848226070404,
|
|
"loss": 0.16081146895885468,
|
|
"loss/core": 0.16081146895885468,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.294299602508545,
|
|
"rewards/margins": 1.2129321098327637,
|
|
"rewards/rejected": 1.0813673734664917,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.00954426151276545,
|
|
"grad_norm": 912.0,
|
|
"ht_mnpo/logit": 0.1957351267337799,
|
|
"ht_mnpo/residual": 0.18823513388633728,
|
|
"ht_mnpo/residual_abs": 0.20851953327655792,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.0424561500549316,
|
|
"logits/rejected": -1.9713897705078125,
|
|
"logps/chosen": -0.27538567781448364,
|
|
"logps/rejected": -0.28329208493232727,
|
|
"loss": 0.67027348279953,
|
|
"loss/core": 0.67027348279953,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.2881271839141846,
|
|
"rewards/margins": 1.9573514461517334,
|
|
"rewards/rejected": 1.3307758569717407,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014316392269148175,
|
|
"grad_norm": 2.28125,
|
|
"ht_mnpo/logit": 0.22769351303577423,
|
|
"ht_mnpo/residual": 0.22019347548484802,
|
|
"ht_mnpo/residual_abs": 0.23837217688560486,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.1736812591552734,
|
|
"logits/rejected": -2.193058967590332,
|
|
"logps/chosen": -0.3110750913619995,
|
|
"logps/rejected": -0.31340306997299194,
|
|
"loss": 0.7280125617980957,
|
|
"loss/core": 0.7280125617980957,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.035675048828125,
|
|
"rewards/margins": 2.27693510055542,
|
|
"rewards/rejected": 0.7587399482727051,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.0190885230255309,
|
|
"grad_norm": 188.0,
|
|
"ht_mnpo/logit": 0.17653341591358185,
|
|
"ht_mnpo/residual": 0.16903340816497803,
|
|
"ht_mnpo/residual_abs": 0.19080820679664612,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -2.2002100944519043,
|
|
"logits/rejected": -2.175318717956543,
|
|
"logps/chosen": -0.30930784344673157,
|
|
"logps/rejected": -0.32532697916030884,
|
|
"loss": 0.6724044680595398,
|
|
"loss/core": 0.6724044680595398,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.882935047149658,
|
|
"rewards/margins": 1.765333890914917,
|
|
"rewards/rejected": 1.117600917816162,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.023860653781913623,
|
|
"grad_norm": 1.5546875,
|
|
"ht_mnpo/logit": 0.13093149662017822,
|
|
"ht_mnpo/residual": 0.1234315037727356,
|
|
"ht_mnpo/residual_abs": 0.14939559996128082,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.1146342754364014,
|
|
"logits/rejected": -2.0983939170837402,
|
|
"logps/chosen": -0.29903319478034973,
|
|
"logps/rejected": -0.29846540093421936,
|
|
"loss": 0.27747631072998047,
|
|
"loss/core": 0.27747631072998047,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.928014039993286,
|
|
"rewards/margins": 1.3093150854110718,
|
|
"rewards/rejected": 1.6186988353729248,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.02863278453829635,
|
|
"grad_norm": 12.8125,
|
|
"ht_mnpo/logit": 0.22172394394874573,
|
|
"ht_mnpo/residual": 0.2142239511013031,
|
|
"ht_mnpo/residual_abs": 0.35139912366867065,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -1.725611686706543,
|
|
"logits/rejected": -1.81534743309021,
|
|
"logps/chosen": -0.2695452570915222,
|
|
"logps/rejected": -0.24922709167003632,
|
|
"loss": 1.0329101085662842,
|
|
"loss/core": 1.0329101085662842,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.7073774337768555,
|
|
"rewards/margins": 2.2172398567199707,
|
|
"rewards/rejected": 2.490137815475464,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.03340491529467907,
|
|
"grad_norm": 158.0,
|
|
"ht_mnpo/logit": -0.06736380606889725,
|
|
"ht_mnpo/residual": -0.07486380636692047,
|
|
"ht_mnpo/residual_abs": 0.2497224360704422,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -2.0232789516448975,
|
|
"logits/rejected": -1.9483411312103271,
|
|
"logps/chosen": -0.30015066266059875,
|
|
"logps/rejected": -0.29495859146118164,
|
|
"loss": 0.734642744064331,
|
|
"loss/core": 0.734642744064331,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.769416093826294,
|
|
"rewards/margins": -0.6736379861831665,
|
|
"rewards/rejected": 2.44305419921875,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.0381770460510618,
|
|
"grad_norm": 22.625,
|
|
"ht_mnpo/logit": 0.06484057754278183,
|
|
"ht_mnpo/residual": 0.05734057351946831,
|
|
"ht_mnpo/residual_abs": 0.25225889682769775,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.049146890640259,
|
|
"logits/rejected": -2.0784709453582764,
|
|
"logps/chosen": -0.2865927219390869,
|
|
"logps/rejected": -0.30831488966941833,
|
|
"loss": 0.7600542306900024,
|
|
"loss/core": 0.7600542306900024,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3934733867645264,
|
|
"rewards/margins": 0.6484058499336243,
|
|
"rewards/rejected": 1.7450675964355469,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04294917680744453,
|
|
"grad_norm": 13.6875,
|
|
"ht_mnpo/logit": 0.08705328404903412,
|
|
"ht_mnpo/residual": 0.07955329120159149,
|
|
"ht_mnpo/residual_abs": 0.145945206284523,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -1.9960081577301025,
|
|
"logits/rejected": -1.9657599925994873,
|
|
"logps/chosen": -0.2881585359573364,
|
|
"logps/rejected": -0.2911851406097412,
|
|
"loss": 0.12006343901157379,
|
|
"loss/core": 0.12006343901157379,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3550467491149902,
|
|
"rewards/margins": 0.8705328106880188,
|
|
"rewards/rejected": 1.4845139980316162,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.047721307563827246,
|
|
"grad_norm": 43.0,
|
|
"ht_mnpo/logit": 0.19554190337657928,
|
|
"ht_mnpo/residual": 0.18804192543029785,
|
|
"ht_mnpo/residual_abs": 0.19585713744163513,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -2.2572484016418457,
|
|
"logits/rejected": -2.201063632965088,
|
|
"logps/chosen": -0.27145764231681824,
|
|
"logps/rejected": -0.2754862606525421,
|
|
"loss": 0.4455382823944092,
|
|
"loss/core": 0.4455382823944092,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.870077133178711,
|
|
"rewards/margins": 1.9554193019866943,
|
|
"rewards/rejected": 0.9146578907966614,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05249343832020997,
|
|
"grad_norm": 10.75,
|
|
"ht_mnpo/logit": 0.09912572801113129,
|
|
"ht_mnpo/residual": 0.09162574261426926,
|
|
"ht_mnpo/residual_abs": 0.10510493814945221,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -2.0953731536865234,
|
|
"logits/rejected": -2.0514919757843018,
|
|
"logps/chosen": -0.2865874171257019,
|
|
"logps/rejected": -0.28994354605674744,
|
|
"loss": 0.11057962477207184,
|
|
"loss/core": 0.11057962477207184,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.285587787628174,
|
|
"rewards/margins": 0.99125736951828,
|
|
"rewards/rejected": 1.2943300008773804,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.0572655690765927,
|
|
"grad_norm": 620.0,
|
|
"ht_mnpo/logit": -0.04805581644177437,
|
|
"ht_mnpo/residual": -0.05555582791566849,
|
|
"ht_mnpo/residual_abs": 0.25350916385650635,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -2.2205891609191895,
|
|
"logits/rejected": -2.147505283355713,
|
|
"logps/chosen": -0.3061199486255646,
|
|
"logps/rejected": -0.30441951751708984,
|
|
"loss": 0.8527676463127136,
|
|
"loss/core": 0.8527676463127136,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.0320839881896973,
|
|
"rewards/margins": -0.48055824637413025,
|
|
"rewards/rejected": 2.5126423835754395,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.062037699832975426,
|
|
"grad_norm": 2.75,
|
|
"ht_mnpo/logit": 0.10299861431121826,
|
|
"ht_mnpo/residual": 0.09549861401319504,
|
|
"ht_mnpo/residual_abs": 0.13395439088344574,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -2.0961742401123047,
|
|
"logits/rejected": -2.160503387451172,
|
|
"logps/chosen": -0.30310171842575073,
|
|
"logps/rejected": -0.28012874722480774,
|
|
"loss": 0.12896214425563812,
|
|
"loss/core": 0.12896214425563812,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2790303230285645,
|
|
"rewards/margins": 1.0299861431121826,
|
|
"rewards/rejected": 1.2490440607070923,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06680983058935815,
|
|
"grad_norm": 50.0,
|
|
"ht_mnpo/logit": 0.0025874082930386066,
|
|
"ht_mnpo/residual": -0.004912590142339468,
|
|
"ht_mnpo/residual_abs": 0.15773822367191315,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -2.150377035140991,
|
|
"logits/rejected": -2.037092924118042,
|
|
"logps/chosen": -0.283245325088501,
|
|
"logps/rejected": -0.28494200110435486,
|
|
"loss": 0.12411870062351227,
|
|
"loss/core": 0.12411870062351227,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.957980751991272,
|
|
"rewards/margins": 0.02587403729557991,
|
|
"rewards/rejected": 1.932106614112854,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07158196134574088,
|
|
"grad_norm": 808.0,
|
|
"ht_mnpo/logit": 0.16834411025047302,
|
|
"ht_mnpo/residual": 0.1608441174030304,
|
|
"ht_mnpo/residual_abs": 0.20434637367725372,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -2.2492072582244873,
|
|
"logits/rejected": -2.1484618186950684,
|
|
"logps/chosen": -0.27808278799057007,
|
|
"logps/rejected": -0.29606252908706665,
|
|
"loss": 0.561755359172821,
|
|
"loss/core": 0.561755359172821,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.157169818878174,
|
|
"rewards/margins": 1.683441162109375,
|
|
"rewards/rejected": 1.473728895187378,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.0763540921021236,
|
|
"grad_norm": 3.265625,
|
|
"ht_mnpo/logit": 0.13237906992435455,
|
|
"ht_mnpo/residual": 0.12487907707691193,
|
|
"ht_mnpo/residual_abs": 0.14425772428512573,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.16731858253479,
|
|
"logits/rejected": -2.2766902446746826,
|
|
"logps/chosen": -0.2905097007751465,
|
|
"logps/rejected": -0.2843090891838074,
|
|
"loss": 0.2487218827009201,
|
|
"loss/core": 0.2487218827009201,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.213404893875122,
|
|
"rewards/margins": 1.3237906694412231,
|
|
"rewards/rejected": 0.889613926410675,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08112622285850632,
|
|
"grad_norm": 0.326171875,
|
|
"ht_mnpo/logit": 0.10314911603927612,
|
|
"ht_mnpo/residual": 0.09564913064241409,
|
|
"ht_mnpo/residual_abs": 0.10694156587123871,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.077899694442749,
|
|
"logits/rejected": -2.1457324028015137,
|
|
"logps/chosen": -0.30360931158065796,
|
|
"logps/rejected": -0.2841089367866516,
|
|
"loss": 0.09745658189058304,
|
|
"loss/core": 0.09745658189058304,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9278924465179443,
|
|
"rewards/margins": 1.0314911603927612,
|
|
"rewards/rejected": 0.8964012265205383,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08589835361488905,
|
|
"grad_norm": 56.5,
|
|
"ht_mnpo/logit": 0.09610621631145477,
|
|
"ht_mnpo/residual": 0.08860620856285095,
|
|
"ht_mnpo/residual_abs": 0.16850264370441437,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.107797145843506,
|
|
"logits/rejected": -2.0631887912750244,
|
|
"logps/chosen": -0.3219011723995209,
|
|
"logps/rejected": -0.2910079061985016,
|
|
"loss": 0.5044264793395996,
|
|
"loss/core": 0.5044264793395996,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2050461769104004,
|
|
"rewards/margins": 0.9610620737075806,
|
|
"rewards/rejected": 1.2439838647842407,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09067048437127177,
|
|
"grad_norm": 616.0,
|
|
"ht_mnpo/logit": 0.028515029698610306,
|
|
"ht_mnpo/residual": 0.021015027537941933,
|
|
"ht_mnpo/residual_abs": 0.14724914729595184,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.2570595741271973,
|
|
"logits/rejected": -2.347463607788086,
|
|
"logps/chosen": -0.32577449083328247,
|
|
"logps/rejected": -0.30825769901275635,
|
|
"loss": 0.38386693596839905,
|
|
"loss/core": 0.38386693596839905,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8190761804580688,
|
|
"rewards/margins": 0.28515028953552246,
|
|
"rewards/rejected": 1.5339261293411255,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09544261512765449,
|
|
"grad_norm": 209.0,
|
|
"ht_mnpo/logit": 0.27891913056373596,
|
|
"ht_mnpo/residual": 0.27141913771629333,
|
|
"ht_mnpo/residual_abs": 0.275497168302536,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -1.7922618389129639,
|
|
"logits/rejected": -1.8475412130355835,
|
|
"logps/chosen": -0.3087330460548401,
|
|
"logps/rejected": -0.2941829562187195,
|
|
"loss": 1.4505915641784668,
|
|
"loss/core": 1.4505915641784668,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.890589952468872,
|
|
"rewards/margins": 2.789191484451294,
|
|
"rewards/rejected": 1.1013987064361572,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10021474588403723,
|
|
"grad_norm": 0.46875,
|
|
"ht_mnpo/logit": 0.09551374614238739,
|
|
"ht_mnpo/residual": 0.08801373094320297,
|
|
"ht_mnpo/residual_abs": 0.20917508006095886,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.0991432666778564,
|
|
"logits/rejected": -2.105532169342041,
|
|
"logps/chosen": -0.3069460988044739,
|
|
"logps/rejected": -0.32189440727233887,
|
|
"loss": 0.5695691108703613,
|
|
"loss/core": 0.5695691108703613,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.5684151649475098,
|
|
"rewards/margins": 0.9551375508308411,
|
|
"rewards/rejected": 1.6132776737213135,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10498687664041995,
|
|
"grad_norm": 0.7265625,
|
|
"ht_mnpo/logit": 0.13444851338863373,
|
|
"ht_mnpo/residual": 0.1269485205411911,
|
|
"ht_mnpo/residual_abs": 0.14191502332687378,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -2.226870059967041,
|
|
"logits/rejected": -2.155787706375122,
|
|
"logps/chosen": -0.30255234241485596,
|
|
"logps/rejected": -0.31519246101379395,
|
|
"loss": 0.32785916328430176,
|
|
"loss/core": 0.32785916328430176,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3320839405059814,
|
|
"rewards/margins": 1.3444854021072388,
|
|
"rewards/rejected": 0.987598717212677,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10975900739680267,
|
|
"grad_norm": 3.9375,
|
|
"ht_mnpo/logit": 0.059714823961257935,
|
|
"ht_mnpo/residual": 0.05221483111381531,
|
|
"ht_mnpo/residual_abs": 0.075630322098732,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -2.2276268005371094,
|
|
"logits/rejected": -2.1443827152252197,
|
|
"logps/chosen": -0.2710269093513489,
|
|
"logps/rejected": -0.2652212679386139,
|
|
"loss": 0.05404385179281235,
|
|
"loss/core": 0.05404385179281235,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.458458662033081,
|
|
"rewards/margins": 0.5971482992172241,
|
|
"rewards/rejected": 0.8613104820251465,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.1145311381531854,
|
|
"grad_norm": 23.25,
|
|
"ht_mnpo/logit": 0.10005225241184235,
|
|
"ht_mnpo/residual": 0.09255225211381912,
|
|
"ht_mnpo/residual_abs": 0.1233731061220169,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -1.990856409072876,
|
|
"logits/rejected": -2.0017800331115723,
|
|
"logps/chosen": -0.2697809338569641,
|
|
"logps/rejected": -0.2667064368724823,
|
|
"loss": 0.08419139683246613,
|
|
"loss/core": 0.08419139683246613,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.219022750854492,
|
|
"rewards/margins": 1.0005223751068115,
|
|
"rewards/rejected": 1.2185002565383911,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11930326890956812,
|
|
"grad_norm": 792.0,
|
|
"ht_mnpo/logit": 0.14017124474048615,
|
|
"ht_mnpo/residual": 0.13267125189304352,
|
|
"ht_mnpo/residual_abs": 0.1821003258228302,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -1.970934510231018,
|
|
"logits/rejected": -2.0060877799987793,
|
|
"logps/chosen": -0.30272430181503296,
|
|
"logps/rejected": -0.3132968842983246,
|
|
"loss": 0.5580740571022034,
|
|
"loss/core": 0.5580740571022034,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.1881790161132812,
|
|
"rewards/margins": 1.401712417602539,
|
|
"rewards/rejected": 1.786466360092163,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.12407539966595085,
|
|
"grad_norm": 2.671875,
|
|
"ht_mnpo/logit": 0.11025525629520416,
|
|
"ht_mnpo/residual": 0.10275526344776154,
|
|
"ht_mnpo/residual_abs": 0.11799643933773041,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -2.1110129356384277,
|
|
"logits/rejected": -2.1078038215637207,
|
|
"logps/chosen": -0.33342495560646057,
|
|
"logps/rejected": -0.3335180878639221,
|
|
"loss": 0.30878788232803345,
|
|
"loss/core": 0.30878788232803345,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.003382682800293,
|
|
"rewards/margins": 1.1025526523590088,
|
|
"rewards/rejected": 0.9008301496505737,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12884753042233357,
|
|
"grad_norm": 98.5,
|
|
"ht_mnpo/logit": 0.010446786880493164,
|
|
"ht_mnpo/residual": 0.0029467791318893433,
|
|
"ht_mnpo/residual_abs": 0.10158848762512207,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.2710258960723877,
|
|
"logits/rejected": -2.1853766441345215,
|
|
"logps/chosen": -0.26459789276123047,
|
|
"logps/rejected": -0.26528677344322205,
|
|
"loss": 0.10957517474889755,
|
|
"loss/core": 0.10957517474889755,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.3557019233703613,
|
|
"rewards/margins": 0.10446784645318985,
|
|
"rewards/rejected": 1.2512340545654297,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.1336196611787163,
|
|
"grad_norm": 284.0,
|
|
"ht_mnpo/logit": 0.12432773411273956,
|
|
"ht_mnpo/residual": 0.11682772636413574,
|
|
"ht_mnpo/residual_abs": 0.16263815760612488,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.1260056495666504,
|
|
"logits/rejected": -2.184141159057617,
|
|
"logps/chosen": -0.2923770844936371,
|
|
"logps/rejected": -0.27389150857925415,
|
|
"loss": 0.3137539029121399,
|
|
"loss/core": 0.3137539029121399,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.815502643585205,
|
|
"rewards/margins": 1.2432773113250732,
|
|
"rewards/rejected": 1.5722254514694214,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.138391791935099,
|
|
"grad_norm": 0.52734375,
|
|
"ht_mnpo/logit": 0.0494002029299736,
|
|
"ht_mnpo/residual": 0.04190019890666008,
|
|
"ht_mnpo/residual_abs": 0.0962391048669815,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.9057468175888062,
|
|
"logits/rejected": -1.8750278949737549,
|
|
"logps/chosen": -0.3120560050010681,
|
|
"logps/rejected": -0.3050037920475006,
|
|
"loss": 0.07905109971761703,
|
|
"loss/core": 0.07905109971761703,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.3618370294570923,
|
|
"rewards/margins": 0.49400201439857483,
|
|
"rewards/rejected": 0.867834746837616,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14316392269148176,
|
|
"grad_norm": 81.0,
|
|
"ht_mnpo/logit": 0.07038834691047668,
|
|
"ht_mnpo/residual": 0.06288834661245346,
|
|
"ht_mnpo/residual_abs": 0.19300627708435059,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.0689001083374023,
|
|
"logits/rejected": -2.040731906890869,
|
|
"logps/chosen": -0.2997715473175049,
|
|
"logps/rejected": -0.31551551818847656,
|
|
"loss": 0.45480862259864807,
|
|
"loss/core": 0.45480862259864807,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.503751516342163,
|
|
"rewards/margins": 0.7038835287094116,
|
|
"rewards/rejected": 1.7998679876327515,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.14793605344786448,
|
|
"grad_norm": 5.21875,
|
|
"ht_mnpo/logit": 0.028568068519234657,
|
|
"ht_mnpo/residual": 0.021068070083856583,
|
|
"ht_mnpo/residual_abs": 0.12228935956954956,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.1313748359680176,
|
|
"logits/rejected": -2.1528689861297607,
|
|
"logps/chosen": -0.28180932998657227,
|
|
"logps/rejected": -0.27187174558639526,
|
|
"loss": 0.2188756912946701,
|
|
"loss/core": 0.2188756912946701,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.679626226425171,
|
|
"rewards/margins": 0.28568071126937866,
|
|
"rewards/rejected": 1.393945574760437,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.1527081842042472,
|
|
"grad_norm": 185.0,
|
|
"ht_mnpo/logit": 0.15196958184242249,
|
|
"ht_mnpo/residual": 0.14446958899497986,
|
|
"ht_mnpo/residual_abs": 0.25919145345687866,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.049853563308716,
|
|
"logits/rejected": -2.1069273948669434,
|
|
"logps/chosen": -0.28914421796798706,
|
|
"logps/rejected": -0.2759200930595398,
|
|
"loss": 0.6769053936004639,
|
|
"loss/core": 0.6769053936004639,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.205308437347412,
|
|
"rewards/margins": 1.51969575881958,
|
|
"rewards/rejected": 1.6856123208999634,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15748031496062992,
|
|
"grad_norm": 2.28125,
|
|
"ht_mnpo/logit": 0.15034791827201843,
|
|
"ht_mnpo/residual": 0.1428479254245758,
|
|
"ht_mnpo/residual_abs": 0.14793327450752258,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.3357603549957275,
|
|
"logits/rejected": -2.300586223602295,
|
|
"logps/chosen": -0.2931155562400818,
|
|
"logps/rejected": -0.27911141514778137,
|
|
"loss": 0.2518713176250458,
|
|
"loss/core": 0.2518713176250458,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.707345485687256,
|
|
"rewards/margins": 1.5034791231155396,
|
|
"rewards/rejected": 1.2038663625717163,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16225244571701264,
|
|
"grad_norm": 1.25,
|
|
"ht_mnpo/logit": 0.031133702024817467,
|
|
"ht_mnpo/residual": 0.023633703589439392,
|
|
"ht_mnpo/residual_abs": 0.09513436257839203,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.1481313705444336,
|
|
"logits/rejected": -2.0097994804382324,
|
|
"logps/chosen": -0.27028369903564453,
|
|
"logps/rejected": -0.2625959813594818,
|
|
"loss": 0.09149707853794098,
|
|
"loss/core": 0.09149707853794098,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7255449295043945,
|
|
"rewards/margins": 0.3113369941711426,
|
|
"rewards/rejected": 1.4142078161239624,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16702457647339536,
|
|
"grad_norm": 27.625,
|
|
"ht_mnpo/logit": 0.07642657309770584,
|
|
"ht_mnpo/residual": 0.06892655789852142,
|
|
"ht_mnpo/residual_abs": 0.07498085498809814,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.273118257522583,
|
|
"logits/rejected": -2.1809802055358887,
|
|
"logps/chosen": -0.28101009130477905,
|
|
"logps/rejected": -0.2591692805290222,
|
|
"loss": 0.06240804120898247,
|
|
"loss/core": 0.06240804120898247,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.3919998407363892,
|
|
"rewards/margins": 0.7642655968666077,
|
|
"rewards/rejected": 0.6277341246604919,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.1717967072297781,
|
|
"grad_norm": 4.59375,
|
|
"ht_mnpo/logit": 0.08522136509418488,
|
|
"ht_mnpo/residual": 0.07772137224674225,
|
|
"ht_mnpo/residual_abs": 0.08434905111789703,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -2.163783550262451,
|
|
"logits/rejected": -2.2361397743225098,
|
|
"logps/chosen": -0.29033127427101135,
|
|
"logps/rejected": -0.27159175276756287,
|
|
"loss": 0.03848674148321152,
|
|
"loss/core": 0.03848674148321152,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.901350975036621,
|
|
"rewards/margins": 0.8522136807441711,
|
|
"rewards/rejected": 1.0491373538970947,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17656883798616083,
|
|
"grad_norm": 2.015625,
|
|
"ht_mnpo/logit": 0.12886855006217957,
|
|
"ht_mnpo/residual": 0.12136852741241455,
|
|
"ht_mnpo/residual_abs": 0.12450716644525528,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -1.9544246196746826,
|
|
"logits/rejected": -2.0396523475646973,
|
|
"logps/chosen": -0.2824137806892395,
|
|
"logps/rejected": -0.2820433974266052,
|
|
"loss": 0.1304716169834137,
|
|
"loss/core": 0.1304716169834137,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1400837898254395,
|
|
"rewards/margins": 1.2886853218078613,
|
|
"rewards/rejected": 0.8513985872268677,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18134096874254355,
|
|
"grad_norm": 2.234375,
|
|
"ht_mnpo/logit": 0.05361006781458855,
|
|
"ht_mnpo/residual": 0.04611007869243622,
|
|
"ht_mnpo/residual_abs": 0.06801304221153259,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -2.177982807159424,
|
|
"logits/rejected": -2.2904984951019287,
|
|
"logps/chosen": -0.29912227392196655,
|
|
"logps/rejected": -0.2927282452583313,
|
|
"loss": 0.02491976134479046,
|
|
"loss/core": 0.02491976134479046,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.471571445465088,
|
|
"rewards/margins": 0.5361008048057556,
|
|
"rewards/rejected": 0.9354707598686218,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18611309949892627,
|
|
"grad_norm": 15.5,
|
|
"ht_mnpo/logit": 0.16110555827617645,
|
|
"ht_mnpo/residual": 0.15360556542873383,
|
|
"ht_mnpo/residual_abs": 0.1779450923204422,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -2.0854153633117676,
|
|
"logits/rejected": -2.057650089263916,
|
|
"logps/chosen": -0.2939828038215637,
|
|
"logps/rejected": -0.2942020893096924,
|
|
"loss": 0.16689643263816833,
|
|
"loss/core": 0.16689643263816833,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.914524793624878,
|
|
"rewards/margins": 1.611055612564087,
|
|
"rewards/rejected": 1.303469181060791,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19088523025530899,
|
|
"grad_norm": 2.859375,
|
|
"ht_mnpo/logit": 0.10722651332616806,
|
|
"ht_mnpo/residual": 0.09972651302814484,
|
|
"ht_mnpo/residual_abs": 0.10703112930059433,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -2.2011075019836426,
|
|
"logits/rejected": -2.1862375736236572,
|
|
"logps/chosen": -0.29882553219795227,
|
|
"logps/rejected": -0.30318737030029297,
|
|
"loss": 0.23726752400398254,
|
|
"loss/core": 0.23726752400398254,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2529048919677734,
|
|
"rewards/margins": 1.072265386581421,
|
|
"rewards/rejected": 1.1806399822235107,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19565736101169173,
|
|
"grad_norm": 11.4375,
|
|
"ht_mnpo/logit": 0.053003937005996704,
|
|
"ht_mnpo/residual": 0.04550394043326378,
|
|
"ht_mnpo/residual_abs": 0.07750043272972107,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -1.9984384775161743,
|
|
"logits/rejected": -2.088771343231201,
|
|
"logps/chosen": -0.3163023591041565,
|
|
"logps/rejected": -0.30901581048965454,
|
|
"loss": 0.04021575301885605,
|
|
"loss/core": 0.04021575301885605,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.633466124534607,
|
|
"rewards/margins": 0.5300394296646118,
|
|
"rewards/rejected": 1.1034266948699951,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20042949176807445,
|
|
"grad_norm": 12.75,
|
|
"ht_mnpo/logit": 0.1604582965373993,
|
|
"ht_mnpo/residual": 0.15295830368995667,
|
|
"ht_mnpo/residual_abs": 0.17036454379558563,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -1.9503463506698608,
|
|
"logits/rejected": -1.9217140674591064,
|
|
"logps/chosen": -0.2880111336708069,
|
|
"logps/rejected": -0.2948765754699707,
|
|
"loss": 0.2796785235404968,
|
|
"loss/core": 0.2796785235404968,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1876330375671387,
|
|
"rewards/margins": 1.6045830249786377,
|
|
"rewards/rejected": 1.5830501317977905,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20520162252445717,
|
|
"grad_norm": 1.0546875,
|
|
"ht_mnpo/logit": 0.084672711789608,
|
|
"ht_mnpo/residual": 0.07717271149158478,
|
|
"ht_mnpo/residual_abs": 0.1215781569480896,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.1271862983703613,
|
|
"logits/rejected": -2.171344041824341,
|
|
"logps/chosen": -0.2948567271232605,
|
|
"logps/rejected": -0.2792275547981262,
|
|
"loss": 0.18875925242900848,
|
|
"loss/core": 0.18875925242900848,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0924696922302246,
|
|
"rewards/margins": 0.8467270135879517,
|
|
"rewards/rejected": 1.2457425594329834,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.2099737532808399,
|
|
"grad_norm": 0.76171875,
|
|
"ht_mnpo/logit": 0.06622101366519928,
|
|
"ht_mnpo/residual": 0.058721017092466354,
|
|
"ht_mnpo/residual_abs": 0.08079180121421814,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -2.091046094894409,
|
|
"logits/rejected": -2.2000317573547363,
|
|
"logps/chosen": -0.32790035009384155,
|
|
"logps/rejected": -0.3206120431423187,
|
|
"loss": 0.08376762270927429,
|
|
"loss/core": 0.08376762270927429,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9445091485977173,
|
|
"rewards/margins": 0.66221022605896,
|
|
"rewards/rejected": 1.2822988033294678,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2147458840372226,
|
|
"grad_norm": 0.94140625,
|
|
"ht_mnpo/logit": 0.15462861955165863,
|
|
"ht_mnpo/residual": 0.1471286118030548,
|
|
"ht_mnpo/residual_abs": 0.17850852012634277,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.1533384323120117,
|
|
"logits/rejected": -2.2266592979431152,
|
|
"logps/chosen": -0.25487151741981506,
|
|
"logps/rejected": -0.23334865272045135,
|
|
"loss": 0.24176493287086487,
|
|
"loss/core": 0.24176493287086487,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.2291808128356934,
|
|
"rewards/margins": 1.5462863445281982,
|
|
"rewards/rejected": 1.6828947067260742,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21951801479360533,
|
|
"grad_norm": 4.625,
|
|
"ht_mnpo/logit": 0.05176293104887009,
|
|
"ht_mnpo/residual": 0.04426293820142746,
|
|
"ht_mnpo/residual_abs": 0.0576145239174366,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -2.076111316680908,
|
|
"logits/rejected": -2.167879581451416,
|
|
"logps/chosen": -0.32002463936805725,
|
|
"logps/rejected": -0.29538848996162415,
|
|
"loss": 0.014069770462810993,
|
|
"loss/core": 0.014069770462810993,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.1483262777328491,
|
|
"rewards/margins": 0.5176293253898621,
|
|
"rewards/rejected": 0.6306968927383423,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.22429014554998808,
|
|
"grad_norm": 4.34375,
|
|
"ht_mnpo/logit": 0.06035786122083664,
|
|
"ht_mnpo/residual": 0.05285785719752312,
|
|
"ht_mnpo/residual_abs": 0.1479470431804657,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.0307183265686035,
|
|
"logits/rejected": -2.164533853530884,
|
|
"logps/chosen": -0.30357375741004944,
|
|
"logps/rejected": -0.30138644576072693,
|
|
"loss": 0.30801814794540405,
|
|
"loss/core": 0.30801814794540405,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.844181776046753,
|
|
"rewards/margins": 0.6035785675048828,
|
|
"rewards/rejected": 1.2406030893325806,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.2290622763063708,
|
|
"grad_norm": 3.0,
|
|
"ht_mnpo/logit": 0.05508917570114136,
|
|
"ht_mnpo/residual": 0.04758917912840843,
|
|
"ht_mnpo/residual_abs": 0.062050361186265945,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.1817893981933594,
|
|
"logits/rejected": -2.206329345703125,
|
|
"logps/chosen": -0.2946377098560333,
|
|
"logps/rejected": -0.2780408263206482,
|
|
"loss": 0.011683464981615543,
|
|
"loss/core": 0.011683464981615543,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5235569477081299,
|
|
"rewards/margins": 0.5508917570114136,
|
|
"rewards/rejected": 0.9726651310920715,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23383440706275352,
|
|
"grad_norm": 2.21875,
|
|
"ht_mnpo/logit": 0.03650509566068649,
|
|
"ht_mnpo/residual": 0.02900509163737297,
|
|
"ht_mnpo/residual_abs": 0.11014048010110855,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.124574899673462,
|
|
"logits/rejected": -2.221391439437866,
|
|
"logps/chosen": -0.2924458384513855,
|
|
"logps/rejected": -0.27032575011253357,
|
|
"loss": 0.13753008842468262,
|
|
"loss/core": 0.13753008842468262,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5719541311264038,
|
|
"rewards/margins": 0.36505094170570374,
|
|
"rewards/rejected": 1.2069032192230225,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23860653781913624,
|
|
"grad_norm": 47.25,
|
|
"ht_mnpo/logit": 0.20946970582008362,
|
|
"ht_mnpo/residual": 0.2019696980714798,
|
|
"ht_mnpo/residual_abs": 0.20413513481616974,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.8938868045806885,
|
|
"logits/rejected": -1.9979091882705688,
|
|
"logps/chosen": -0.3038504123687744,
|
|
"logps/rejected": -0.2779809534549713,
|
|
"loss": 0.3277028501033783,
|
|
"loss/core": 0.3277028501033783,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.351757526397705,
|
|
"rewards/margins": 2.0946967601776123,
|
|
"rewards/rejected": 1.2570606470108032,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24337866857551896,
|
|
"grad_norm": 4.5625,
|
|
"ht_mnpo/logit": 0.10556943714618683,
|
|
"ht_mnpo/residual": 0.0980694368481636,
|
|
"ht_mnpo/residual_abs": 0.13419580459594727,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -1.8236820697784424,
|
|
"logits/rejected": -1.8897043466567993,
|
|
"logps/chosen": -0.298318088054657,
|
|
"logps/rejected": -0.3006042242050171,
|
|
"loss": 0.1540885865688324,
|
|
"loss/core": 0.1540885865688324,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6379857063293457,
|
|
"rewards/margins": 1.055694341659546,
|
|
"rewards/rejected": 1.5822908878326416,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2481507993319017,
|
|
"grad_norm": 1416.0,
|
|
"ht_mnpo/logit": 0.21744589507579803,
|
|
"ht_mnpo/residual": 0.2099459171295166,
|
|
"ht_mnpo/residual_abs": 0.2343859225511551,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -2.1703085899353027,
|
|
"logits/rejected": -2.276538372039795,
|
|
"logps/chosen": -0.30491483211517334,
|
|
"logps/rejected": -0.2938116788864136,
|
|
"loss": 1.0860064029693604,
|
|
"loss/core": 1.0860064029693604,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2028307914733887,
|
|
"rewards/margins": 2.1744587421417236,
|
|
"rewards/rejected": 1.028372049331665,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.2529229300882844,
|
|
"grad_norm": 33.25,
|
|
"ht_mnpo/logit": 0.03215058147907257,
|
|
"ht_mnpo/residual": 0.0246505755931139,
|
|
"ht_mnpo/residual_abs": 0.195493683218956,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.069486141204834,
|
|
"logits/rejected": -2.1000301837921143,
|
|
"logps/chosen": -0.28381848335266113,
|
|
"logps/rejected": -0.2866091728210449,
|
|
"loss": 0.5351065397262573,
|
|
"loss/core": 0.5351065397262573,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2341432571411133,
|
|
"rewards/margins": 0.3215058445930481,
|
|
"rewards/rejected": 1.912637710571289,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.25769506084466715,
|
|
"grad_norm": 207.0,
|
|
"ht_mnpo/logit": 0.032596416771411896,
|
|
"ht_mnpo/residual": 0.025096416473388672,
|
|
"ht_mnpo/residual_abs": 0.2024488002061844,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -2.0715479850769043,
|
|
"logits/rejected": -2.1490814685821533,
|
|
"logps/chosen": -0.29047778248786926,
|
|
"logps/rejected": -0.3188256025314331,
|
|
"loss": 0.5344532132148743,
|
|
"loss/core": 0.5344532132148743,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0660104751586914,
|
|
"rewards/margins": 0.3259641230106354,
|
|
"rewards/rejected": 1.740046501159668,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.26246719160104987,
|
|
"grad_norm": 6.84375,
|
|
"ht_mnpo/logit": 0.0682886391878128,
|
|
"ht_mnpo/residual": 0.06078863888978958,
|
|
"ht_mnpo/residual_abs": 0.09558697044849396,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.068638324737549,
|
|
"logits/rejected": -2.1749560832977295,
|
|
"logps/chosen": -0.29401373863220215,
|
|
"logps/rejected": -0.278691828250885,
|
|
"loss": 0.05401838943362236,
|
|
"loss/core": 0.05401838943362236,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9330079555511475,
|
|
"rewards/margins": 0.6828864812850952,
|
|
"rewards/rejected": 1.250121831893921,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.2672393223574326,
|
|
"grad_norm": 98.0,
|
|
"ht_mnpo/logit": 0.11557488143444061,
|
|
"ht_mnpo/residual": 0.10807488113641739,
|
|
"ht_mnpo/residual_abs": 0.22392027080059052,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.0900626182556152,
|
|
"logits/rejected": -2.1388344764709473,
|
|
"logps/chosen": -0.30804580450057983,
|
|
"logps/rejected": -0.30006811022758484,
|
|
"loss": 0.4923165738582611,
|
|
"loss/core": 0.4923165738582611,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.7162301540374756,
|
|
"rewards/margins": 1.155748963356018,
|
|
"rewards/rejected": 1.560481309890747,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2720114531138153,
|
|
"grad_norm": 1.671875,
|
|
"ht_mnpo/logit": 0.007366104517132044,
|
|
"ht_mnpo/residual": -0.00013389457308221608,
|
|
"ht_mnpo/residual_abs": 0.10880545526742935,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -2.2310268878936768,
|
|
"logits/rejected": -2.114835262298584,
|
|
"logps/chosen": -0.29032495617866516,
|
|
"logps/rejected": -0.3384189009666443,
|
|
"loss": 0.08280204236507416,
|
|
"loss/core": 0.08280204236507416,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.7172960042953491,
|
|
"rewards/margins": 0.07366105169057846,
|
|
"rewards/rejected": 1.6436350345611572,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.276783583870198,
|
|
"grad_norm": 44.75,
|
|
"ht_mnpo/logit": 0.06611434370279312,
|
|
"ht_mnpo/residual": 0.0586143359541893,
|
|
"ht_mnpo/residual_abs": 0.07593923807144165,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -2.26133394241333,
|
|
"logits/rejected": -2.217014789581299,
|
|
"logps/chosen": -0.3368655741214752,
|
|
"logps/rejected": -0.3344263434410095,
|
|
"loss": 0.10027849674224854,
|
|
"loss/core": 0.10027849674224854,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.0940282344818115,
|
|
"rewards/margins": 0.6611433625221252,
|
|
"rewards/rejected": 0.4328847825527191,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28155571462658074,
|
|
"grad_norm": 147.0,
|
|
"ht_mnpo/logit": 0.13316653668880463,
|
|
"ht_mnpo/residual": 0.125666543841362,
|
|
"ht_mnpo/residual_abs": 0.14974115788936615,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.052053689956665,
|
|
"logits/rejected": -2.149163007736206,
|
|
"logps/chosen": -0.30694466829299927,
|
|
"logps/rejected": -0.2819734215736389,
|
|
"loss": 0.289144366979599,
|
|
"loss/core": 0.289144366979599,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.095669746398926,
|
|
"rewards/margins": 1.3316655158996582,
|
|
"rewards/rejected": 0.7640042901039124,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.2863278453829635,
|
|
"grad_norm": 5.84375,
|
|
"ht_mnpo/logit": 0.09821562469005585,
|
|
"ht_mnpo/residual": 0.09071563184261322,
|
|
"ht_mnpo/residual_abs": 0.1706034541130066,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -2.054267168045044,
|
|
"logits/rejected": -2.129446506500244,
|
|
"logps/chosen": -0.2598414719104767,
|
|
"logps/rejected": -0.25766080617904663,
|
|
"loss": 0.1819196343421936,
|
|
"loss/core": 0.1819196343421936,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9472148418426514,
|
|
"rewards/margins": 0.9821560978889465,
|
|
"rewards/rejected": 1.96505868434906,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.29109997613934624,
|
|
"grad_norm": 6.3125,
|
|
"ht_mnpo/logit": 0.08229192346334457,
|
|
"ht_mnpo/residual": 0.07479192316532135,
|
|
"ht_mnpo/residual_abs": 0.08601853251457214,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -1.9865754842758179,
|
|
"logits/rejected": -2.013413429260254,
|
|
"logps/chosen": -0.3034398555755615,
|
|
"logps/rejected": -0.3090708255767822,
|
|
"loss": 0.037676356732845306,
|
|
"loss/core": 0.037676356732845306,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2789227962493896,
|
|
"rewards/margins": 0.8229190707206726,
|
|
"rewards/rejected": 1.4560037851333618,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.29587210689572896,
|
|
"grad_norm": 524.0,
|
|
"ht_mnpo/logit": 0.09432446211576462,
|
|
"ht_mnpo/residual": 0.0868244618177414,
|
|
"ht_mnpo/residual_abs": 0.22896528244018555,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -1.9858556985855103,
|
|
"logits/rejected": -1.9708316326141357,
|
|
"logps/chosen": -0.28563809394836426,
|
|
"logps/rejected": -0.29865071177482605,
|
|
"loss": 0.6932410597801208,
|
|
"loss/core": 0.6932410597801208,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.816760540008545,
|
|
"rewards/margins": 0.9432447552680969,
|
|
"rewards/rejected": 1.8735158443450928,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.3006442376521117,
|
|
"grad_norm": 46.75,
|
|
"ht_mnpo/logit": 0.0416237898170948,
|
|
"ht_mnpo/residual": 0.03412378951907158,
|
|
"ht_mnpo/residual_abs": 0.15009064972400665,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -2.215585231781006,
|
|
"logits/rejected": -2.188673496246338,
|
|
"logps/chosen": -0.31491440534591675,
|
|
"logps/rejected": -0.2965540885925293,
|
|
"loss": 0.20979872345924377,
|
|
"loss/core": 0.20979872345924377,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8721401691436768,
|
|
"rewards/margins": 0.4162379205226898,
|
|
"rewards/rejected": 1.4559022188186646,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3054163684084944,
|
|
"grad_norm": 21.125,
|
|
"ht_mnpo/logit": 0.10669668018817902,
|
|
"ht_mnpo/residual": 0.09919667989015579,
|
|
"ht_mnpo/residual_abs": 0.13064010441303253,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -2.0546205043792725,
|
|
"logits/rejected": -1.999367356300354,
|
|
"logps/chosen": -0.28702324628829956,
|
|
"logps/rejected": -0.2923528552055359,
|
|
"loss": 0.08457665145397186,
|
|
"loss/core": 0.08457665145397186,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4331345558166504,
|
|
"rewards/margins": 1.0669667720794678,
|
|
"rewards/rejected": 1.366167664527893,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.3101884991648771,
|
|
"grad_norm": 4.21875,
|
|
"ht_mnpo/logit": 0.09450945258140564,
|
|
"ht_mnpo/residual": 0.08700944483280182,
|
|
"ht_mnpo/residual_abs": 0.09081277996301651,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -1.9875255823135376,
|
|
"logits/rejected": -2.1427712440490723,
|
|
"logps/chosen": -0.29921954870224,
|
|
"logps/rejected": -0.2899077832698822,
|
|
"loss": 0.07706113159656525,
|
|
"loss/core": 0.07706113159656525,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.703743577003479,
|
|
"rewards/margins": 0.9450944662094116,
|
|
"rewards/rejected": 0.7586489319801331,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31496062992125984,
|
|
"grad_norm": 3.0,
|
|
"ht_mnpo/logit": 0.07892023026943207,
|
|
"ht_mnpo/residual": 0.07142021507024765,
|
|
"ht_mnpo/residual_abs": 0.08247885853052139,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.087024211883545,
|
|
"logits/rejected": -2.1432571411132812,
|
|
"logps/chosen": -0.3117908835411072,
|
|
"logps/rejected": -0.2991257607936859,
|
|
"loss": 0.06879410147666931,
|
|
"loss/core": 0.06879410147666931,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.5523658990859985,
|
|
"rewards/margins": 0.7892022132873535,
|
|
"rewards/rejected": 0.7631637454032898,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.31973276067764256,
|
|
"grad_norm": 1.578125,
|
|
"ht_mnpo/logit": 0.100077785551548,
|
|
"ht_mnpo/residual": 0.09257779270410538,
|
|
"ht_mnpo/residual_abs": 0.15659311413764954,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.0467448234558105,
|
|
"logits/rejected": -1.9432637691497803,
|
|
"logps/chosen": -0.3127209544181824,
|
|
"logps/rejected": -0.3082302212715149,
|
|
"loss": 0.19428648054599762,
|
|
"loss/core": 0.19428648054599762,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.354982852935791,
|
|
"rewards/margins": 1.0007779598236084,
|
|
"rewards/rejected": 1.354204773902893,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3245048914340253,
|
|
"grad_norm": 1.1484375,
|
|
"ht_mnpo/logit": 0.011610162444412708,
|
|
"ht_mnpo/residual": 0.004110163543373346,
|
|
"ht_mnpo/residual_abs": 0.14836415648460388,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -2.0763049125671387,
|
|
"logits/rejected": -2.044464111328125,
|
|
"logps/chosen": -0.31426161527633667,
|
|
"logps/rejected": -0.3251628875732422,
|
|
"loss": 0.24287572503089905,
|
|
"loss/core": 0.24287572503089905,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9379974603652954,
|
|
"rewards/margins": 0.11610164493322372,
|
|
"rewards/rejected": 1.821895956993103,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.329277022190408,
|
|
"grad_norm": 139.0,
|
|
"ht_mnpo/logit": 0.14599408209323883,
|
|
"ht_mnpo/residual": 0.1384941041469574,
|
|
"ht_mnpo/residual_abs": 0.22581353783607483,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -1.9784504175186157,
|
|
"logits/rejected": -2.0479235649108887,
|
|
"logps/chosen": -0.3185192346572876,
|
|
"logps/rejected": -0.2856663763523102,
|
|
"loss": 0.476330041885376,
|
|
"loss/core": 0.476330041885376,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1129283905029297,
|
|
"rewards/margins": 1.4599411487579346,
|
|
"rewards/rejected": 1.6529871225357056,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.3340491529467907,
|
|
"grad_norm": 121.5,
|
|
"ht_mnpo/logit": 0.17999222874641418,
|
|
"ht_mnpo/residual": 0.17249223589897156,
|
|
"ht_mnpo/residual_abs": 0.1953013390302658,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.32238507270813,
|
|
"logits/rejected": -2.366455078125,
|
|
"logps/chosen": -0.2689044177532196,
|
|
"logps/rejected": -0.2708611488342285,
|
|
"loss": 0.45748645067214966,
|
|
"loss/core": 0.45748645067214966,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1744790077209473,
|
|
"rewards/margins": 1.799922227859497,
|
|
"rewards/rejected": 1.3745567798614502,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3388212837031735,
|
|
"grad_norm": 772.0,
|
|
"ht_mnpo/logit": 0.06556814908981323,
|
|
"ht_mnpo/residual": 0.058068156242370605,
|
|
"ht_mnpo/residual_abs": 0.15479174256324768,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -2.154000997543335,
|
|
"logits/rejected": -2.145259141921997,
|
|
"logps/chosen": -0.2757951617240906,
|
|
"logps/rejected": -0.27257150411605835,
|
|
"loss": 0.21897700428962708,
|
|
"loss/core": 0.21897700428962708,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.779329538345337,
|
|
"rewards/margins": 0.6556814908981323,
|
|
"rewards/rejected": 2.123647928237915,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.3435934144595562,
|
|
"grad_norm": 776.0,
|
|
"ht_mnpo/logit": 0.1693677008152008,
|
|
"ht_mnpo/residual": 0.16186770796775818,
|
|
"ht_mnpo/residual_abs": 0.19508880376815796,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -1.8399804830551147,
|
|
"logits/rejected": -1.8790878057479858,
|
|
"logps/chosen": -0.296386182308197,
|
|
"logps/rejected": -0.31673291325569153,
|
|
"loss": 0.43224653601646423,
|
|
"loss/core": 0.43224653601646423,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0844686031341553,
|
|
"rewards/margins": 1.6936769485473633,
|
|
"rewards/rejected": 1.390791654586792,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34836554521593893,
|
|
"grad_norm": 27.75,
|
|
"ht_mnpo/logit": 0.04867623373866081,
|
|
"ht_mnpo/residual": 0.04117623716592789,
|
|
"ht_mnpo/residual_abs": 0.04830501973628998,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -2.164468765258789,
|
|
"logits/rejected": -2.201307773590088,
|
|
"logps/chosen": -0.2984023094177246,
|
|
"logps/rejected": -0.29477953910827637,
|
|
"loss": 0.021916760131716728,
|
|
"loss/core": 0.021916760131716728,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.2955366373062134,
|
|
"rewards/margins": 0.4867623746395111,
|
|
"rewards/rejected": 0.8087741732597351,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.35313767597232165,
|
|
"grad_norm": 7.90625,
|
|
"ht_mnpo/logit": 0.07078660279512405,
|
|
"ht_mnpo/residual": 0.06328661739826202,
|
|
"ht_mnpo/residual_abs": 0.19043698906898499,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.1319940090179443,
|
|
"logits/rejected": -2.105229616165161,
|
|
"logps/chosen": -0.2718384563922882,
|
|
"logps/rejected": -0.26169219613075256,
|
|
"loss": 0.5162893533706665,
|
|
"loss/core": 0.5162893533706665,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.2212352752685547,
|
|
"rewards/margins": 0.7078661918640137,
|
|
"rewards/rejected": 1.5133693218231201,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35790980672870437,
|
|
"grad_norm": 25.125,
|
|
"ht_mnpo/logit": 0.13367705047130585,
|
|
"ht_mnpo/residual": 0.12617704272270203,
|
|
"ht_mnpo/residual_abs": 0.16303515434265137,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -2.170954465866089,
|
|
"logits/rejected": -2.2756118774414062,
|
|
"logps/chosen": -0.259833425283432,
|
|
"logps/rejected": -0.2742316722869873,
|
|
"loss": 0.2501140236854553,
|
|
"loss/core": 0.2501140236854553,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.697831869125366,
|
|
"rewards/margins": 1.3367704153060913,
|
|
"rewards/rejected": 1.3610615730285645,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.3626819374850871,
|
|
"grad_norm": 8.0625,
|
|
"ht_mnpo/logit": 0.24639646708965302,
|
|
"ht_mnpo/residual": 0.2388964593410492,
|
|
"ht_mnpo/residual_abs": 0.250093549489975,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -1.9891340732574463,
|
|
"logits/rejected": -2.001060962677002,
|
|
"logps/chosen": -0.33623114228248596,
|
|
"logps/rejected": -0.2880457043647766,
|
|
"loss": 0.6706468462944031,
|
|
"loss/core": 0.6706468462944031,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.5934009552001953,
|
|
"rewards/margins": 2.4639649391174316,
|
|
"rewards/rejected": 1.1294362545013428,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.3674540682414698,
|
|
"grad_norm": 16.125,
|
|
"ht_mnpo/logit": 0.07829246670007706,
|
|
"ht_mnpo/residual": 0.07079247385263443,
|
|
"ht_mnpo/residual_abs": 0.0979897528886795,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -2.110302686691284,
|
|
"logits/rejected": -2.122204303741455,
|
|
"logps/chosen": -0.2984893321990967,
|
|
"logps/rejected": -0.2900088429450989,
|
|
"loss": 0.058689720928668976,
|
|
"loss/core": 0.058689720928668976,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8839489221572876,
|
|
"rewards/margins": 0.7829247117042542,
|
|
"rewards/rejected": 1.1010242700576782,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.37222619899785253,
|
|
"grad_norm": 3.921875,
|
|
"ht_mnpo/logit": 0.0583811029791832,
|
|
"ht_mnpo/residual": 0.05088110640645027,
|
|
"ht_mnpo/residual_abs": 0.11353194713592529,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -2.2036306858062744,
|
|
"logits/rejected": -2.26830792427063,
|
|
"logps/chosen": -0.28720030188560486,
|
|
"logps/rejected": -0.29090017080307007,
|
|
"loss": 0.10279025137424469,
|
|
"loss/core": 0.10279025137424469,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0018017292022705,
|
|
"rewards/margins": 0.5838111042976379,
|
|
"rewards/rejected": 1.4179904460906982,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37699832975423525,
|
|
"grad_norm": 231.0,
|
|
"ht_mnpo/logit": 0.23413963615894318,
|
|
"ht_mnpo/residual": 0.22663962841033936,
|
|
"ht_mnpo/residual_abs": 0.2551414966583252,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -2.015768051147461,
|
|
"logits/rejected": -2.0945444107055664,
|
|
"logps/chosen": -0.27710121870040894,
|
|
"logps/rejected": -0.2746776044368744,
|
|
"loss": 0.4984128475189209,
|
|
"loss/core": 0.4984128475189209,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 4.204476356506348,
|
|
"rewards/margins": 2.3413960933685303,
|
|
"rewards/rejected": 1.863080620765686,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38177046051061797,
|
|
"grad_norm": 4.53125,
|
|
"ht_mnpo/logit": 0.09178599715232849,
|
|
"ht_mnpo/residual": 0.08428598940372467,
|
|
"ht_mnpo/residual_abs": 0.09066607058048248,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.175234317779541,
|
|
"logits/rejected": -2.227020025253296,
|
|
"logps/chosen": -0.3004263937473297,
|
|
"logps/rejected": -0.2858661711215973,
|
|
"loss": 0.07631814479827881,
|
|
"loss/core": 0.07631814479827881,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.299661636352539,
|
|
"rewards/margins": 0.9178601503372192,
|
|
"rewards/rejected": 1.3818013668060303,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.3865425912670007,
|
|
"grad_norm": 9.1875,
|
|
"ht_mnpo/logit": 0.1291302740573883,
|
|
"ht_mnpo/residual": 0.12163027375936508,
|
|
"ht_mnpo/residual_abs": 0.1272890567779541,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -2.067502737045288,
|
|
"logits/rejected": -2.1148393154144287,
|
|
"logps/chosen": -0.2825814187526703,
|
|
"logps/rejected": -0.2730342745780945,
|
|
"loss": 0.23866212368011475,
|
|
"loss/core": 0.23866212368011475,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0164496898651123,
|
|
"rewards/margins": 1.2913026809692383,
|
|
"rewards/rejected": 0.7251466512680054,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39131472202338347,
|
|
"grad_norm": 17.25,
|
|
"ht_mnpo/logit": 0.07708557695150375,
|
|
"ht_mnpo/residual": 0.06958558410406113,
|
|
"ht_mnpo/residual_abs": 0.1799410581588745,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.0323519706726074,
|
|
"logits/rejected": -2.03326678276062,
|
|
"logps/chosen": -0.29431599378585815,
|
|
"logps/rejected": -0.29893699288368225,
|
|
"loss": 0.2613184452056885,
|
|
"loss/core": 0.2613184452056885,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.014284133911133,
|
|
"rewards/margins": 0.7708557844161987,
|
|
"rewards/rejected": 1.2434285879135132,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3960868527797662,
|
|
"grad_norm": 3.421875,
|
|
"ht_mnpo/logit": 0.18855980038642883,
|
|
"ht_mnpo/residual": 0.18105976283550262,
|
|
"ht_mnpo/residual_abs": 0.19883492588996887,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -1.9817771911621094,
|
|
"logits/rejected": -2.0678811073303223,
|
|
"logps/chosen": -0.3057534992694855,
|
|
"logps/rejected": -0.2749868631362915,
|
|
"loss": 0.6331941485404968,
|
|
"loss/core": 0.6331941485404968,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.0550625324249268,
|
|
"rewards/margins": 1.8855979442596436,
|
|
"rewards/rejected": 1.1694644689559937,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.4008589835361489,
|
|
"grad_norm": 21.75,
|
|
"ht_mnpo/logit": 0.13565869629383087,
|
|
"ht_mnpo/residual": 0.12815868854522705,
|
|
"ht_mnpo/residual_abs": 0.13334575295448303,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -2.0041332244873047,
|
|
"logits/rejected": -2.0797832012176514,
|
|
"logps/chosen": -0.2827741801738739,
|
|
"logps/rejected": -0.269229531288147,
|
|
"loss": 0.09638514369726181,
|
|
"loss/core": 0.09638514369726181,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.2633628845214844,
|
|
"rewards/margins": 1.3565869331359863,
|
|
"rewards/rejected": 0.906775951385498,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4056311142925316,
|
|
"grad_norm": 896.0,
|
|
"ht_mnpo/logit": 0.15091469883918762,
|
|
"ht_mnpo/residual": 0.143414705991745,
|
|
"ht_mnpo/residual_abs": 0.16229206323623657,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -2.006406784057617,
|
|
"logits/rejected": -2.115539073944092,
|
|
"logps/chosen": -0.29817336797714233,
|
|
"logps/rejected": -0.28226596117019653,
|
|
"loss": 0.5429047346115112,
|
|
"loss/core": 0.5429047346115112,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.30177640914917,
|
|
"rewards/margins": 1.509147047996521,
|
|
"rewards/rejected": 0.7926293611526489,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41040324504891434,
|
|
"grad_norm": 138.0,
|
|
"ht_mnpo/logit": 0.09805858135223389,
|
|
"ht_mnpo/residual": 0.09055860340595245,
|
|
"ht_mnpo/residual_abs": 0.1379367709159851,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.42768931388855,
|
|
"logits/rejected": -2.3442633152008057,
|
|
"logps/chosen": -0.28302669525146484,
|
|
"logps/rejected": -0.28555774688720703,
|
|
"loss": 0.2293454110622406,
|
|
"loss/core": 0.2293454110622406,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.095808267593384,
|
|
"rewards/margins": 0.9805858731269836,
|
|
"rewards/rejected": 1.1152220964431763,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41517537580529706,
|
|
"grad_norm": 160.0,
|
|
"ht_mnpo/logit": 0.1208895891904831,
|
|
"ht_mnpo/residual": 0.11338958889245987,
|
|
"ht_mnpo/residual_abs": 0.20339921116828918,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.232008457183838,
|
|
"logits/rejected": -2.221562147140503,
|
|
"logps/chosen": -0.2803706228733063,
|
|
"logps/rejected": -0.29765453934669495,
|
|
"loss": 0.9300113916397095,
|
|
"loss/core": 0.9300113916397095,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4720828533172607,
|
|
"rewards/margins": 1.2088956832885742,
|
|
"rewards/rejected": 1.263187289237976,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4199475065616798,
|
|
"grad_norm": 25.75,
|
|
"ht_mnpo/logit": 0.09195519983768463,
|
|
"ht_mnpo/residual": 0.08445519208908081,
|
|
"ht_mnpo/residual_abs": 0.12991644442081451,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -2.333383083343506,
|
|
"logits/rejected": -2.346522808074951,
|
|
"logps/chosen": -0.29035550355911255,
|
|
"logps/rejected": -0.27913159132003784,
|
|
"loss": 0.1505986452102661,
|
|
"loss/core": 0.1505986452102661,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3275554180145264,
|
|
"rewards/margins": 0.9195519685745239,
|
|
"rewards/rejected": 1.4080034494400024,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.4247196373180625,
|
|
"grad_norm": 7.0,
|
|
"ht_mnpo/logit": 0.05209118872880936,
|
|
"ht_mnpo/residual": 0.04459118843078613,
|
|
"ht_mnpo/residual_abs": 0.04903823882341385,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -2.1833479404449463,
|
|
"logits/rejected": -2.244410991668701,
|
|
"logps/chosen": -0.29939165711402893,
|
|
"logps/rejected": -0.29696738719940186,
|
|
"loss": 0.013744374737143517,
|
|
"loss/core": 0.013744374737143517,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2123219966888428,
|
|
"rewards/margins": 0.5209118723869324,
|
|
"rewards/rejected": 0.6914102435112,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4294917680744452,
|
|
"grad_norm": 9.875,
|
|
"ht_mnpo/logit": 0.0671422928571701,
|
|
"ht_mnpo/residual": 0.05964229628443718,
|
|
"ht_mnpo/residual_abs": 0.06930109858512878,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -2.143707752227783,
|
|
"logits/rejected": -2.1564431190490723,
|
|
"logps/chosen": -0.28366950154304504,
|
|
"logps/rejected": -0.27332574129104614,
|
|
"loss": 0.024859780445694923,
|
|
"loss/core": 0.024859780445694923,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.8562815189361572,
|
|
"rewards/margins": 0.6714230179786682,
|
|
"rewards/rejected": 1.1848584413528442,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.43426389883082794,
|
|
"grad_norm": 0.65234375,
|
|
"ht_mnpo/logit": 0.1936154067516327,
|
|
"ht_mnpo/residual": 0.18611541390419006,
|
|
"ht_mnpo/residual_abs": 0.19285865128040314,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -2.1005420684814453,
|
|
"logits/rejected": -2.149547815322876,
|
|
"logps/chosen": -0.29087719321250916,
|
|
"logps/rejected": -0.2866514027118683,
|
|
"loss": 0.6541997194290161,
|
|
"loss/core": 0.6541997194290161,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9990909099578857,
|
|
"rewards/margins": 1.9361540079116821,
|
|
"rewards/rejected": 1.0629371404647827,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.43903602958721066,
|
|
"grad_norm": 2592.0,
|
|
"ht_mnpo/logit": 0.19766196608543396,
|
|
"ht_mnpo/residual": 0.19016198813915253,
|
|
"ht_mnpo/residual_abs": 0.22098055481910706,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -2.2931854724884033,
|
|
"logits/rejected": -2.2046098709106445,
|
|
"logps/chosen": -0.2994667887687683,
|
|
"logps/rejected": -0.28606337308883667,
|
|
"loss": 0.7362050414085388,
|
|
"loss/core": 0.7362050414085388,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.860424518585205,
|
|
"rewards/margins": 1.9766197204589844,
|
|
"rewards/rejected": 0.8838049173355103,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44380816034359344,
|
|
"grad_norm": 10.625,
|
|
"ht_mnpo/logit": 0.06282283365726471,
|
|
"ht_mnpo/residual": 0.05532282590866089,
|
|
"ht_mnpo/residual_abs": 0.07460145652294159,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -2.1819441318511963,
|
|
"logits/rejected": -2.180514097213745,
|
|
"logps/chosen": -0.2965949773788452,
|
|
"logps/rejected": -0.278251975774765,
|
|
"loss": 0.026971453800797462,
|
|
"loss/core": 0.026971453800797462,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5738455057144165,
|
|
"rewards/margins": 0.6282283067703247,
|
|
"rewards/rejected": 0.9456171989440918,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.44858029109997616,
|
|
"grad_norm": 19.875,
|
|
"ht_mnpo/logit": 0.14918012917041779,
|
|
"ht_mnpo/residual": 0.14168013632297516,
|
|
"ht_mnpo/residual_abs": 0.22854459285736084,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -1.9400546550750732,
|
|
"logits/rejected": -2.032510280609131,
|
|
"logps/chosen": -0.2952631115913391,
|
|
"logps/rejected": -0.28796467185020447,
|
|
"loss": 0.5161349773406982,
|
|
"loss/core": 0.5161349773406982,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.023547410964966,
|
|
"rewards/margins": 1.491801381111145,
|
|
"rewards/rejected": 1.5317460298538208,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.4533524218563589,
|
|
"grad_norm": 46.25,
|
|
"ht_mnpo/logit": 0.13631603121757507,
|
|
"ht_mnpo/residual": 0.12881603837013245,
|
|
"ht_mnpo/residual_abs": 0.14233455061912537,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.1496260166168213,
|
|
"logits/rejected": -2.2138235569000244,
|
|
"logps/chosen": -0.3058208227157593,
|
|
"logps/rejected": -0.2893269956111908,
|
|
"loss": 0.4681578576564789,
|
|
"loss/core": 0.4681578576564789,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2276856899261475,
|
|
"rewards/margins": 1.3631603717803955,
|
|
"rewards/rejected": 0.8645254969596863,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.4581245526127416,
|
|
"grad_norm": 2.03125,
|
|
"ht_mnpo/logit": 0.07291718572378159,
|
|
"ht_mnpo/residual": 0.06541718542575836,
|
|
"ht_mnpo/residual_abs": 0.1957278698682785,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -2.0847127437591553,
|
|
"logits/rejected": -2.0919432640075684,
|
|
"logps/chosen": -0.28979983925819397,
|
|
"logps/rejected": -0.28023532032966614,
|
|
"loss": 0.4742603898048401,
|
|
"loss/core": 0.4742603898048401,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.021890163421631,
|
|
"rewards/margins": 0.729171872138977,
|
|
"rewards/rejected": 1.2927181720733643,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4628966833691243,
|
|
"grad_norm": 1144.0,
|
|
"ht_mnpo/logit": 0.2463802993297577,
|
|
"ht_mnpo/residual": 0.2388802468776703,
|
|
"ht_mnpo/residual_abs": 0.245798259973526,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.0273923873901367,
|
|
"logits/rejected": -1.9866002798080444,
|
|
"logps/chosen": -0.3362005949020386,
|
|
"logps/rejected": -0.3115374445915222,
|
|
"loss": 0.9858028292655945,
|
|
"loss/core": 0.9858028292655945,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.876636028289795,
|
|
"rewards/margins": 2.4638028144836426,
|
|
"rewards/rejected": 1.4128332138061523,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.46766881412550704,
|
|
"grad_norm": 1.6953125,
|
|
"ht_mnpo/logit": 0.059139739722013474,
|
|
"ht_mnpo/residual": 0.05163973569869995,
|
|
"ht_mnpo/residual_abs": 0.10399965941905975,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.0965423583984375,
|
|
"logits/rejected": -2.182716131210327,
|
|
"logps/chosen": -0.29576700925827026,
|
|
"logps/rejected": -0.29857736825942993,
|
|
"loss": 0.09599469602108002,
|
|
"loss/core": 0.09599469602108002,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.5863193273544312,
|
|
"rewards/margins": 0.5913974046707153,
|
|
"rewards/rejected": 0.994921863079071,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47244094488188976,
|
|
"grad_norm": 7.25,
|
|
"ht_mnpo/logit": 0.08595691621303558,
|
|
"ht_mnpo/residual": 0.07845690846443176,
|
|
"ht_mnpo/residual_abs": 0.08687671273946762,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.202465057373047,
|
|
"logits/rejected": -2.1843132972717285,
|
|
"logps/chosen": -0.2824229300022125,
|
|
"logps/rejected": -0.2738740146160126,
|
|
"loss": 0.03743022680282593,
|
|
"loss/core": 0.03743022680282593,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5005347728729248,
|
|
"rewards/margins": 0.8595690727233887,
|
|
"rewards/rejected": 0.6409655809402466,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.4772130756382725,
|
|
"grad_norm": 47.0,
|
|
"ht_mnpo/logit": 0.09756086021661758,
|
|
"ht_mnpo/residual": 0.09006085246801376,
|
|
"ht_mnpo/residual_abs": 0.09450768679380417,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -2.0701684951782227,
|
|
"logits/rejected": -2.1218984127044678,
|
|
"logps/chosen": -0.3352174162864685,
|
|
"logps/rejected": -0.31724342703819275,
|
|
"loss": 0.12419148534536362,
|
|
"loss/core": 0.12419148534536362,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.500693440437317,
|
|
"rewards/margins": 0.9756084680557251,
|
|
"rewards/rejected": 0.5250848531723022,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4819852063946552,
|
|
"grad_norm": 3.40625,
|
|
"ht_mnpo/logit": 0.1528806984424591,
|
|
"ht_mnpo/residual": 0.1453806608915329,
|
|
"ht_mnpo/residual_abs": 0.15676525235176086,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -2.2806396484375,
|
|
"logits/rejected": -2.3282206058502197,
|
|
"logps/chosen": -0.2664569020271301,
|
|
"logps/rejected": -0.2485734224319458,
|
|
"loss": 0.3381979465484619,
|
|
"loss/core": 0.3381979465484619,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9908111095428467,
|
|
"rewards/margins": 1.5288066864013672,
|
|
"rewards/rejected": 1.4620041847229004,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4867573371510379,
|
|
"grad_norm": 1.5546875,
|
|
"ht_mnpo/logit": 0.034343037754297256,
|
|
"ht_mnpo/residual": 0.026843031868338585,
|
|
"ht_mnpo/residual_abs": 0.10809679329395294,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -2.1005702018737793,
|
|
"logits/rejected": -2.0844995975494385,
|
|
"logps/chosen": -0.31387194991111755,
|
|
"logps/rejected": -0.30737537145614624,
|
|
"loss": 0.09971658885478973,
|
|
"loss/core": 0.09971658885478973,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6584968566894531,
|
|
"rewards/margins": 0.34343039989471436,
|
|
"rewards/rejected": 1.3150662183761597,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49152946790742064,
|
|
"grad_norm": 4.90625,
|
|
"ht_mnpo/logit": 0.19290807843208313,
|
|
"ht_mnpo/residual": 0.1854081153869629,
|
|
"ht_mnpo/residual_abs": 0.2561078667640686,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -2.0163683891296387,
|
|
"logits/rejected": -2.0174050331115723,
|
|
"logps/chosen": -0.27022209763526917,
|
|
"logps/rejected": -0.26961493492126465,
|
|
"loss": 0.6102972030639648,
|
|
"loss/core": 0.6102972030639648,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.291226863861084,
|
|
"rewards/margins": 1.9290809631347656,
|
|
"rewards/rejected": 1.3621461391448975,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4963015986638034,
|
|
"grad_norm": 7.5625,
|
|
"ht_mnpo/logit": -0.005236223340034485,
|
|
"ht_mnpo/residual": -0.012736232951283455,
|
|
"ht_mnpo/residual_abs": 0.1328015923500061,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -2.0806922912597656,
|
|
"logits/rejected": -2.0068271160125732,
|
|
"logps/chosen": -0.31831881403923035,
|
|
"logps/rejected": -0.2931275963783264,
|
|
"loss": 0.354873389005661,
|
|
"loss/core": 0.354873389005661,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4059890508651733,
|
|
"rewards/margins": -0.05236220359802246,
|
|
"rewards/rejected": 1.4583513736724854,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5010737294201861,
|
|
"grad_norm": 0.7734375,
|
|
"ht_mnpo/logit": 0.02382918819785118,
|
|
"ht_mnpo/residual": 0.016329189762473106,
|
|
"ht_mnpo/residual_abs": 0.060878295451402664,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.3115806579589844,
|
|
"logits/rejected": -2.299849271774292,
|
|
"logps/chosen": -0.31554344296455383,
|
|
"logps/rejected": -0.3013135492801666,
|
|
"loss": 0.026647498831152916,
|
|
"loss/core": 0.026647498831152916,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.0067657232284546,
|
|
"rewards/margins": 0.2382918894290924,
|
|
"rewards/rejected": 0.7684738636016846,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5058458601765689,
|
|
"grad_norm": 318.0,
|
|
"ht_mnpo/logit": 0.047034695744514465,
|
|
"ht_mnpo/residual": 0.03953469544649124,
|
|
"ht_mnpo/residual_abs": 0.1397225260734558,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.1985840797424316,
|
|
"logits/rejected": -2.074636936187744,
|
|
"logps/chosen": -0.29173484444618225,
|
|
"logps/rejected": -0.2891838550567627,
|
|
"loss": 0.18388794362545013,
|
|
"loss/core": 0.18388794362545013,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.232212543487549,
|
|
"rewards/margins": 0.47034698724746704,
|
|
"rewards/rejected": 1.7618656158447266,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5106179909329516,
|
|
"grad_norm": 1152.0,
|
|
"ht_mnpo/logit": 0.2150212824344635,
|
|
"ht_mnpo/residual": 0.20752127468585968,
|
|
"ht_mnpo/residual_abs": 0.25212562084198,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -2.056133270263672,
|
|
"logits/rejected": -2.110259771347046,
|
|
"logps/chosen": -0.29885774850845337,
|
|
"logps/rejected": -0.29431480169296265,
|
|
"loss": 1.2166507244110107,
|
|
"loss/core": 1.2166507244110107,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.6530518531799316,
|
|
"rewards/margins": 2.1502127647399902,
|
|
"rewards/rejected": 1.502839207649231,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5153901216893343,
|
|
"grad_norm": 1.9375,
|
|
"ht_mnpo/logit": 0.05426005274057388,
|
|
"ht_mnpo/residual": 0.046760059893131256,
|
|
"ht_mnpo/residual_abs": 0.074722521007061,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.156520128250122,
|
|
"logits/rejected": -2.1915793418884277,
|
|
"logps/chosen": -0.2928999066352844,
|
|
"logps/rejected": -0.2622906565666199,
|
|
"loss": 0.0269025769084692,
|
|
"loss/core": 0.0269025769084692,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6733510494232178,
|
|
"rewards/margins": 0.5426005125045776,
|
|
"rewards/rejected": 1.1307506561279297,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520162252445717,
|
|
"grad_norm": 0.66796875,
|
|
"ht_mnpo/logit": 0.10010049492120743,
|
|
"ht_mnpo/residual": 0.09260048717260361,
|
|
"ht_mnpo/residual_abs": 0.10867872089147568,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.0413730144500732,
|
|
"logits/rejected": -2.0470833778381348,
|
|
"logps/chosen": -0.29208695888519287,
|
|
"logps/rejected": -0.27415579557418823,
|
|
"loss": 0.20062501728534698,
|
|
"loss/core": 0.20062501728534698,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.9059289693832397,
|
|
"rewards/margins": 1.001004934310913,
|
|
"rewards/rejected": 0.9049240946769714,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5249343832020997,
|
|
"grad_norm": 1.8671875,
|
|
"ht_mnpo/logit": 0.098787322640419,
|
|
"ht_mnpo/residual": 0.09128732234239578,
|
|
"ht_mnpo/residual_abs": 0.13674144446849823,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -2.286104440689087,
|
|
"logits/rejected": -2.361344814300537,
|
|
"logps/chosen": -0.3042377531528473,
|
|
"logps/rejected": -0.2910940945148468,
|
|
"loss": 0.19538818299770355,
|
|
"loss/core": 0.19538818299770355,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.1192712783813477,
|
|
"rewards/margins": 0.9878730773925781,
|
|
"rewards/rejected": 1.1313982009887695,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5297065139584825,
|
|
"grad_norm": 9.6875,
|
|
"ht_mnpo/logit": 0.08020411431789398,
|
|
"ht_mnpo/residual": 0.07270411401987076,
|
|
"ht_mnpo/residual_abs": 0.0984831228852272,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.1592469215393066,
|
|
"logits/rejected": -2.134700298309326,
|
|
"logps/chosen": -0.25345703959465027,
|
|
"logps/rejected": -0.2741752564907074,
|
|
"loss": 0.06742379814386368,
|
|
"loss/core": 0.06742379814386368,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.854957938194275,
|
|
"rewards/margins": 0.8020410537719727,
|
|
"rewards/rejected": 1.0529168844223022,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5344786447148652,
|
|
"grad_norm": 24.875,
|
|
"ht_mnpo/logit": 0.21764996647834778,
|
|
"ht_mnpo/residual": 0.21014995872974396,
|
|
"ht_mnpo/residual_abs": 0.2130311280488968,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -2.1484036445617676,
|
|
"logits/rejected": -2.2150402069091797,
|
|
"logps/chosen": -0.28861570358276367,
|
|
"logps/rejected": -0.26910072565078735,
|
|
"loss": 0.6849492192268372,
|
|
"loss/core": 0.6849492192268372,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.4669604301452637,
|
|
"rewards/margins": 2.176499605178833,
|
|
"rewards/rejected": 1.2904607057571411,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5392507754712479,
|
|
"grad_norm": 1512.0,
|
|
"ht_mnpo/logit": 0.23818376660346985,
|
|
"ht_mnpo/residual": 0.23068372905254364,
|
|
"ht_mnpo/residual_abs": 0.33580002188682556,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.1061885356903076,
|
|
"logits/rejected": -2.1180262565612793,
|
|
"logps/chosen": -0.2766849994659424,
|
|
"logps/rejected": -0.30740275979042053,
|
|
"loss": 1.4853737354278564,
|
|
"loss/core": 1.4853737354278564,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.7866311073303223,
|
|
"rewards/margins": 2.3818376064300537,
|
|
"rewards/rejected": 1.4047935009002686,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5440229062276306,
|
|
"grad_norm": 0.78125,
|
|
"ht_mnpo/logit": 0.22942838072776794,
|
|
"ht_mnpo/residual": 0.2219284027814865,
|
|
"ht_mnpo/residual_abs": 0.23399917781352997,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -2.007561206817627,
|
|
"logits/rejected": -1.9956079721450806,
|
|
"logps/chosen": -0.2868313193321228,
|
|
"logps/rejected": -0.29199647903442383,
|
|
"loss": 0.5726346373558044,
|
|
"loss/core": 0.5726346373558044,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.5619232654571533,
|
|
"rewards/margins": 2.2942841053009033,
|
|
"rewards/rejected": 1.2676398754119873,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5487950369840133,
|
|
"grad_norm": 2.484375,
|
|
"ht_mnpo/logit": 0.11159713566303253,
|
|
"ht_mnpo/residual": 0.1040971428155899,
|
|
"ht_mnpo/residual_abs": 0.11690263450145721,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.1647210121154785,
|
|
"logits/rejected": -2.207653522491455,
|
|
"logps/chosen": -0.3100038468837738,
|
|
"logps/rejected": -0.281381219625473,
|
|
"loss": 0.09815619885921478,
|
|
"loss/core": 0.09815619885921478,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.275163173675537,
|
|
"rewards/margins": 1.115971326828003,
|
|
"rewards/rejected": 1.1591918468475342,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.553567167740396,
|
|
"grad_norm": 2.921875,
|
|
"ht_mnpo/logit": 0.11486697196960449,
|
|
"ht_mnpo/residual": 0.10736697912216187,
|
|
"ht_mnpo/residual_abs": 0.13199064135551453,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.228494167327881,
|
|
"logits/rejected": -2.189988613128662,
|
|
"logps/chosen": -0.2966822683811188,
|
|
"logps/rejected": -0.2899606227874756,
|
|
"loss": 0.288084477186203,
|
|
"loss/core": 0.288084477186203,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.918851613998413,
|
|
"rewards/margins": 1.1486698389053345,
|
|
"rewards/rejected": 0.7701820135116577,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5583392984967788,
|
|
"grad_norm": 0.6328125,
|
|
"ht_mnpo/logit": 0.06170004606246948,
|
|
"ht_mnpo/residual": 0.054200053215026855,
|
|
"ht_mnpo/residual_abs": 0.09687240421772003,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -1.9388682842254639,
|
|
"logits/rejected": -1.8519554138183594,
|
|
"logps/chosen": -0.26918455958366394,
|
|
"logps/rejected": -0.2541769742965698,
|
|
"loss": 0.042456772178411484,
|
|
"loss/core": 0.042456772178411484,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1972382068634033,
|
|
"rewards/margins": 0.6170004606246948,
|
|
"rewards/rejected": 1.5802377462387085,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5631114292531615,
|
|
"grad_norm": 8.75,
|
|
"ht_mnpo/logit": 0.058099836111068726,
|
|
"ht_mnpo/residual": 0.0505998432636261,
|
|
"ht_mnpo/residual_abs": 0.17364242672920227,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -2.2340292930603027,
|
|
"logits/rejected": -2.1745638847351074,
|
|
"logps/chosen": -0.23693934082984924,
|
|
"logps/rejected": -0.24809077382087708,
|
|
"loss": 0.3619423806667328,
|
|
"loss/core": 0.3619423806667328,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.183443546295166,
|
|
"rewards/margins": 0.5809983611106873,
|
|
"rewards/rejected": 1.6024450063705444,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5678835600095442,
|
|
"grad_norm": 0.1298828125,
|
|
"ht_mnpo/logit": 0.07216361910104752,
|
|
"ht_mnpo/residual": 0.06466361880302429,
|
|
"ht_mnpo/residual_abs": 0.0867505595088005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -2.1809277534484863,
|
|
"logits/rejected": -2.264443874359131,
|
|
"logps/chosen": -0.27737170457839966,
|
|
"logps/rejected": -0.27399271726608276,
|
|
"loss": 0.04572673514485359,
|
|
"loss/core": 0.04572673514485359,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.4415251016616821,
|
|
"rewards/margins": 0.721636176109314,
|
|
"rewards/rejected": 0.7198889851570129,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.572655690765927,
|
|
"grad_norm": 8.8125,
|
|
"ht_mnpo/logit": 0.06845931708812714,
|
|
"ht_mnpo/residual": 0.06095932796597481,
|
|
"ht_mnpo/residual_abs": 0.28927868604660034,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -2.0192017555236816,
|
|
"logits/rejected": -2.1130526065826416,
|
|
"logps/chosen": -0.27419912815093994,
|
|
"logps/rejected": -0.2707653343677521,
|
|
"loss": 1.1791174411773682,
|
|
"loss/core": 1.1791174411773682,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.887829542160034,
|
|
"rewards/margins": 0.6845930814743042,
|
|
"rewards/rejected": 2.2032363414764404,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5774278215223098,
|
|
"grad_norm": 99.5,
|
|
"ht_mnpo/logit": 0.18931171298027039,
|
|
"ht_mnpo/residual": 0.18181169033050537,
|
|
"ht_mnpo/residual_abs": 0.20922286808490753,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -2.2182424068450928,
|
|
"logits/rejected": -2.2502996921539307,
|
|
"logps/chosen": -0.3130815327167511,
|
|
"logps/rejected": -0.3144187033176422,
|
|
"loss": 0.5771145820617676,
|
|
"loss/core": 0.5771145820617676,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.197204113006592,
|
|
"rewards/margins": 1.8931169509887695,
|
|
"rewards/rejected": 1.304086685180664,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5821999522786925,
|
|
"grad_norm": 2.578125,
|
|
"ht_mnpo/logit": 0.09359414875507355,
|
|
"ht_mnpo/residual": 0.08609413355588913,
|
|
"ht_mnpo/residual_abs": 0.10207605361938477,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -2.1247410774230957,
|
|
"logits/rejected": -2.108510971069336,
|
|
"logps/chosen": -0.272658109664917,
|
|
"logps/rejected": -0.28190943598747253,
|
|
"loss": 0.11088033020496368,
|
|
"loss/core": 0.11088033020496368,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.923109769821167,
|
|
"rewards/margins": 0.9359413981437683,
|
|
"rewards/rejected": 0.9871681332588196,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.5869720830350752,
|
|
"grad_norm": 59.75,
|
|
"ht_mnpo/logit": 0.21152541041374207,
|
|
"ht_mnpo/residual": 0.20402541756629944,
|
|
"ht_mnpo/residual_abs": 0.22103989124298096,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -2.192009449005127,
|
|
"logits/rejected": -2.2331554889678955,
|
|
"logps/chosen": -0.31632620096206665,
|
|
"logps/rejected": -0.28495582938194275,
|
|
"loss": 0.684616208076477,
|
|
"loss/core": 0.684616208076477,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1536481380462646,
|
|
"rewards/margins": 2.1152539253234863,
|
|
"rewards/rejected": 1.0383942127227783,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5917442137914579,
|
|
"grad_norm": 34.5,
|
|
"ht_mnpo/logit": 0.043183762580156326,
|
|
"ht_mnpo/residual": 0.0356837622821331,
|
|
"ht_mnpo/residual_abs": 0.1497439444065094,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.1083340644836426,
|
|
"logits/rejected": -2.179131269454956,
|
|
"logps/chosen": -0.26702603697776794,
|
|
"logps/rejected": -0.26075607538223267,
|
|
"loss": 0.13401201367378235,
|
|
"loss/core": 0.13401201367378235,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8873205184936523,
|
|
"rewards/margins": 0.43183764815330505,
|
|
"rewards/rejected": 1.455482840538025,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5965163445478406,
|
|
"grad_norm": 5.53125,
|
|
"ht_mnpo/logit": 0.20867642760276794,
|
|
"ht_mnpo/residual": 0.2011764496564865,
|
|
"ht_mnpo/residual_abs": 0.2585950195789337,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.7806930541992188,
|
|
"logits/rejected": -1.8166608810424805,
|
|
"logps/chosen": -0.24558083713054657,
|
|
"logps/rejected": -0.24023476243019104,
|
|
"loss": 0.46875572204589844,
|
|
"loss/core": 0.46875572204589844,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.4086103439331055,
|
|
"rewards/margins": 2.086764335632324,
|
|
"rewards/rejected": 2.3218460083007812,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6012884753042234,
|
|
"grad_norm": 2.046875,
|
|
"ht_mnpo/logit": 0.13468870520591736,
|
|
"ht_mnpo/residual": 0.12718871235847473,
|
|
"ht_mnpo/residual_abs": 0.1308780014514923,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -2.0953762531280518,
|
|
"logits/rejected": -2.1909890174865723,
|
|
"logps/chosen": -0.26866987347602844,
|
|
"logps/rejected": -0.2617102563381195,
|
|
"loss": 0.21047914028167725,
|
|
"loss/core": 0.21047914028167725,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.607395648956299,
|
|
"rewards/margins": 1.3468869924545288,
|
|
"rewards/rejected": 1.2605082988739014,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6060606060606061,
|
|
"grad_norm": 16.25,
|
|
"ht_mnpo/logit": 0.046742431819438934,
|
|
"ht_mnpo/residual": 0.03924242779612541,
|
|
"ht_mnpo/residual_abs": 0.11967098712921143,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -2.2242093086242676,
|
|
"logits/rejected": -2.122405529022217,
|
|
"logps/chosen": -0.30006808042526245,
|
|
"logps/rejected": -0.28186583518981934,
|
|
"loss": 0.1131725087761879,
|
|
"loss/core": 0.1131725087761879,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.4187710285186768,
|
|
"rewards/margins": 0.46742430329322815,
|
|
"rewards/rejected": 0.9513468742370605,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6108327368169888,
|
|
"grad_norm": 52.5,
|
|
"ht_mnpo/logit": 0.033292241394519806,
|
|
"ht_mnpo/residual": 0.02579224482178688,
|
|
"ht_mnpo/residual_abs": 0.2367764711380005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.073786973953247,
|
|
"logits/rejected": -2.0959436893463135,
|
|
"logps/chosen": -0.28610923886299133,
|
|
"logps/rejected": -0.2980540096759796,
|
|
"loss": 0.48102641105651855,
|
|
"loss/core": 0.48102641105651855,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3441481590270996,
|
|
"rewards/margins": 0.3329223096370697,
|
|
"rewards/rejected": 2.011225938796997,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6156048675733715,
|
|
"grad_norm": 2.9375,
|
|
"ht_mnpo/logit": 0.17397519946098328,
|
|
"ht_mnpo/residual": 0.16647520661354065,
|
|
"ht_mnpo/residual_abs": 0.1911737620830536,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -2.155489444732666,
|
|
"logits/rejected": -2.164496898651123,
|
|
"logps/chosen": -0.29790198802948,
|
|
"logps/rejected": -0.2998579144477844,
|
|
"loss": 0.49661874771118164,
|
|
"loss/core": 0.49661874771118164,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.293790340423584,
|
|
"rewards/margins": 1.7397520542144775,
|
|
"rewards/rejected": 1.554038405418396,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6203769983297542,
|
|
"grad_norm": 32.5,
|
|
"ht_mnpo/logit": 0.1089751347899437,
|
|
"ht_mnpo/residual": 0.10147513449192047,
|
|
"ht_mnpo/residual_abs": 0.11368437111377716,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -2.090285301208496,
|
|
"logits/rejected": -2.1761794090270996,
|
|
"logps/chosen": -0.27180442214012146,
|
|
"logps/rejected": -0.26983508467674255,
|
|
"loss": 0.077135369181633,
|
|
"loss/core": 0.077135369181633,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2141337394714355,
|
|
"rewards/margins": 1.0897514820098877,
|
|
"rewards/rejected": 1.1243823766708374,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.625149129086137,
|
|
"grad_norm": 880.0,
|
|
"ht_mnpo/logit": 0.2785004675388336,
|
|
"ht_mnpo/residual": 0.271000474691391,
|
|
"ht_mnpo/residual_abs": 0.3150801658630371,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.0350708961486816,
|
|
"logits/rejected": -2.0822062492370605,
|
|
"logps/chosen": -0.3091558516025543,
|
|
"logps/rejected": -0.31957852840423584,
|
|
"loss": 1.4940129518508911,
|
|
"loss/core": 1.4940129518508911,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.898334503173828,
|
|
"rewards/margins": 2.7850046157836914,
|
|
"rewards/rejected": 1.113329529762268,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6299212598425197,
|
|
"grad_norm": 16.875,
|
|
"ht_mnpo/logit": 0.17702892422676086,
|
|
"ht_mnpo/residual": 0.16952893137931824,
|
|
"ht_mnpo/residual_abs": 0.1766408532857895,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -1.9041671752929688,
|
|
"logits/rejected": -1.929474115371704,
|
|
"logps/chosen": -0.2832731306552887,
|
|
"logps/rejected": -0.2771525979042053,
|
|
"loss": 0.39101579785346985,
|
|
"loss/core": 0.39101579785346985,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.728006362915039,
|
|
"rewards/margins": 1.770289421081543,
|
|
"rewards/rejected": 0.9577168226242065,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6346933905989024,
|
|
"grad_norm": 15.6875,
|
|
"ht_mnpo/logit": 0.09807918965816498,
|
|
"ht_mnpo/residual": 0.09057918936014175,
|
|
"ht_mnpo/residual_abs": 0.12041660398244858,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -2.069812536239624,
|
|
"logits/rejected": -2.125847816467285,
|
|
"logps/chosen": -0.2814692556858063,
|
|
"logps/rejected": -0.2710791230201721,
|
|
"loss": 0.08960843086242676,
|
|
"loss/core": 0.08960843086242676,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.962667465209961,
|
|
"rewards/margins": 0.9807918667793274,
|
|
"rewards/rejected": 0.9818755388259888,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.6394655213552851,
|
|
"grad_norm": 94.0,
|
|
"ht_mnpo/logit": 0.00905347429215908,
|
|
"ht_mnpo/residual": 0.0015534699195995927,
|
|
"ht_mnpo/residual_abs": 0.147975355386734,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -2.445849657058716,
|
|
"logits/rejected": -2.4024407863616943,
|
|
"logps/chosen": -0.28253886103630066,
|
|
"logps/rejected": -0.2934015989303589,
|
|
"loss": 0.2624434232711792,
|
|
"loss/core": 0.2624434232711792,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.5216253995895386,
|
|
"rewards/margins": 0.09053464233875275,
|
|
"rewards/rejected": 1.4310907125473022,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6442376521116678,
|
|
"grad_norm": 87.0,
|
|
"ht_mnpo/logit": 0.07526589930057526,
|
|
"ht_mnpo/residual": 0.06776590645313263,
|
|
"ht_mnpo/residual_abs": 0.19357003271579742,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.027789831161499,
|
|
"logits/rejected": -2.080132484436035,
|
|
"logps/chosen": -0.2811906039714813,
|
|
"logps/rejected": -0.2772737741470337,
|
|
"loss": 0.30826523900032043,
|
|
"loss/core": 0.30826523900032043,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5021486282348633,
|
|
"rewards/margins": 0.7526589632034302,
|
|
"rewards/rejected": 1.7494895458221436,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6490097828680506,
|
|
"grad_norm": 12.625,
|
|
"ht_mnpo/logit": 0.10573352873325348,
|
|
"ht_mnpo/residual": 0.09823354333639145,
|
|
"ht_mnpo/residual_abs": 0.10365482419729233,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -1.910638451576233,
|
|
"logits/rejected": -1.9565751552581787,
|
|
"logps/chosen": -0.3004496693611145,
|
|
"logps/rejected": -0.2888430058956146,
|
|
"loss": 0.08603724092245102,
|
|
"loss/core": 0.08603724092245102,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.726280927658081,
|
|
"rewards/margins": 1.0573352575302124,
|
|
"rewards/rejected": 1.6689456701278687,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.6537819136244333,
|
|
"grad_norm": 15.6875,
|
|
"ht_mnpo/logit": 0.11498228460550308,
|
|
"ht_mnpo/residual": 0.10748227685689926,
|
|
"ht_mnpo/residual_abs": 0.15228202939033508,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -2.176805019378662,
|
|
"logits/rejected": -2.210144519805908,
|
|
"logps/chosen": -0.2797221839427948,
|
|
"logps/rejected": -0.3026573657989502,
|
|
"loss": 0.24593576788902283,
|
|
"loss/core": 0.24593576788902283,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.403441905975342,
|
|
"rewards/margins": 1.1498225927352905,
|
|
"rewards/rejected": 1.2536190748214722,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.658554044380816,
|
|
"grad_norm": 240.0,
|
|
"ht_mnpo/logit": 0.11702848970890045,
|
|
"ht_mnpo/residual": 0.10952849686145782,
|
|
"ht_mnpo/residual_abs": 0.1646626889705658,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -2.0042319297790527,
|
|
"logits/rejected": -1.979111671447754,
|
|
"logps/chosen": -0.3079618811607361,
|
|
"logps/rejected": -0.2868824601173401,
|
|
"loss": 0.22790667414665222,
|
|
"loss/core": 0.22790667414665222,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4165444374084473,
|
|
"rewards/margins": 1.1702848672866821,
|
|
"rewards/rejected": 1.2462592124938965,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6633261751371987,
|
|
"grad_norm": 984.0,
|
|
"ht_mnpo/logit": 0.18550536036491394,
|
|
"ht_mnpo/residual": 0.1780053675174713,
|
|
"ht_mnpo/residual_abs": 0.1904396116733551,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -2.0949904918670654,
|
|
"logits/rejected": -2.118269205093384,
|
|
"logps/chosen": -0.2962483763694763,
|
|
"logps/rejected": -0.2824075222015381,
|
|
"loss": 0.8668164014816284,
|
|
"loss/core": 0.8668164014816284,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9470839500427246,
|
|
"rewards/margins": 1.8550535440444946,
|
|
"rewards/rejected": 1.0920301675796509,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6680983058935814,
|
|
"grad_norm": 756.0,
|
|
"ht_mnpo/logit": 0.22229436039924622,
|
|
"ht_mnpo/residual": 0.21479432284832,
|
|
"ht_mnpo/residual_abs": 0.28674596548080444,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -2.061213731765747,
|
|
"logits/rejected": -2.021340847015381,
|
|
"logps/chosen": -0.30653488636016846,
|
|
"logps/rejected": -0.3151229918003082,
|
|
"loss": 0.881302535533905,
|
|
"loss/core": 0.881302535533905,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.028003692626953,
|
|
"rewards/margins": 2.2229433059692383,
|
|
"rewards/rejected": 1.8050603866577148,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6728704366499642,
|
|
"grad_norm": 1.5390625,
|
|
"ht_mnpo/logit": 0.0755210891366005,
|
|
"ht_mnpo/residual": 0.06802109628915787,
|
|
"ht_mnpo/residual_abs": 0.15567728877067566,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -2.249558687210083,
|
|
"logits/rejected": -2.192272663116455,
|
|
"logps/chosen": -0.2608739733695984,
|
|
"logps/rejected": -0.27803075313568115,
|
|
"loss": 0.30238884687423706,
|
|
"loss/core": 0.30238884687423706,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7999625205993652,
|
|
"rewards/margins": 0.7552109956741333,
|
|
"rewards/rejected": 2.0447516441345215,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.677642567406347,
|
|
"grad_norm": 414.0,
|
|
"ht_mnpo/logit": 0.11202869564294815,
|
|
"ht_mnpo/residual": 0.10452868789434433,
|
|
"ht_mnpo/residual_abs": 0.25466471910476685,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -2.04231333732605,
|
|
"logits/rejected": -2.1298766136169434,
|
|
"logps/chosen": -0.3229544162750244,
|
|
"logps/rejected": -0.2944870591163635,
|
|
"loss": 0.6831266283988953,
|
|
"loss/core": 0.6831266283988953,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0203940868377686,
|
|
"rewards/margins": 1.1202871799468994,
|
|
"rewards/rejected": 1.9001071453094482,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6824146981627297,
|
|
"grad_norm": 11.75,
|
|
"ht_mnpo/logit": 0.16674119234085083,
|
|
"ht_mnpo/residual": 0.15924116969108582,
|
|
"ht_mnpo/residual_abs": 0.1635814756155014,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -1.7993494272232056,
|
|
"logits/rejected": -1.8742660284042358,
|
|
"logps/chosen": -0.2802506387233734,
|
|
"logps/rejected": -0.30492326617240906,
|
|
"loss": 0.26578038930892944,
|
|
"loss/core": 0.26578038930892944,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.977574110031128,
|
|
"rewards/margins": 1.6674118041992188,
|
|
"rewards/rejected": 1.3101623058319092,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6871868289191124,
|
|
"grad_norm": 2.625,
|
|
"ht_mnpo/logit": 0.21696901321411133,
|
|
"ht_mnpo/residual": 0.2094690054655075,
|
|
"ht_mnpo/residual_abs": 0.21472272276878357,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -2.0282702445983887,
|
|
"logits/rejected": -2.018631935119629,
|
|
"logps/chosen": -0.29306745529174805,
|
|
"logps/rejected": -0.2762381434440613,
|
|
"loss": 0.5803450345993042,
|
|
"loss/core": 0.5803450345993042,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3107528686523438,
|
|
"rewards/margins": 2.169689893722534,
|
|
"rewards/rejected": 1.1410627365112305,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6919589596754951,
|
|
"grad_norm": 1.5078125,
|
|
"ht_mnpo/logit": 0.09387997537851334,
|
|
"ht_mnpo/residual": 0.08637996762990952,
|
|
"ht_mnpo/residual_abs": 0.19136810302734375,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -2.2025420665740967,
|
|
"logits/rejected": -2.1641955375671387,
|
|
"logps/chosen": -0.2895504832267761,
|
|
"logps/rejected": -0.29813945293426514,
|
|
"loss": 0.2689235210418701,
|
|
"loss/core": 0.2689235210418701,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.5900564193725586,
|
|
"rewards/margins": 0.9387996792793274,
|
|
"rewards/rejected": 1.6512569189071655,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6967310904318779,
|
|
"grad_norm": 334.0,
|
|
"ht_mnpo/logit": 0.10368786752223969,
|
|
"ht_mnpo/residual": 0.09618787467479706,
|
|
"ht_mnpo/residual_abs": 0.11059357970952988,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.100865364074707,
|
|
"logits/rejected": -2.1864678859710693,
|
|
"logps/chosen": -0.3016471862792969,
|
|
"logps/rejected": -0.28621745109558105,
|
|
"loss": 0.18038377165794373,
|
|
"loss/core": 0.18038377165794373,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1753203868865967,
|
|
"rewards/margins": 1.0368787050247192,
|
|
"rewards/rejected": 1.138441801071167,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7015032211882606,
|
|
"grad_norm": 153.0,
|
|
"ht_mnpo/logit": -0.06999675184488297,
|
|
"ht_mnpo/residual": -0.07749674469232559,
|
|
"ht_mnpo/residual_abs": 0.15613941848278046,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -2.235835552215576,
|
|
"logits/rejected": -2.1666109561920166,
|
|
"logps/chosen": -0.3065755069255829,
|
|
"logps/rejected": -0.323150098323822,
|
|
"loss": 0.2582674026489258,
|
|
"loss/core": 0.2582674026489258,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.2052594423294067,
|
|
"rewards/margins": -0.6999675035476685,
|
|
"rewards/rejected": 1.9052270650863647,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.7062753519446433,
|
|
"grad_norm": 9.5,
|
|
"ht_mnpo/logit": 0.08259505033493042,
|
|
"ht_mnpo/residual": 0.07509505748748779,
|
|
"ht_mnpo/residual_abs": 0.21390943229198456,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -2.0588953495025635,
|
|
"logits/rejected": -1.9913275241851807,
|
|
"logps/chosen": -0.266545832157135,
|
|
"logps/rejected": -0.2904679477214813,
|
|
"loss": 0.3255937695503235,
|
|
"loss/core": 0.3255937695503235,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.877443790435791,
|
|
"rewards/margins": 0.8259506225585938,
|
|
"rewards/rejected": 2.0514931678771973,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.711047482701026,
|
|
"grad_norm": 23.0,
|
|
"ht_mnpo/logit": 0.15350745618343353,
|
|
"ht_mnpo/residual": 0.1460074782371521,
|
|
"ht_mnpo/residual_abs": 0.15337680280208588,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -2.0869879722595215,
|
|
"logits/rejected": -2.1158251762390137,
|
|
"logps/chosen": -0.27177831530570984,
|
|
"logps/rejected": -0.27791404724121094,
|
|
"loss": 0.2072025090456009,
|
|
"loss/core": 0.2072025090456009,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.250014543533325,
|
|
"rewards/margins": 1.5350747108459473,
|
|
"rewards/rejected": 0.7149398922920227,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7158196134574087,
|
|
"grad_norm": 14.1875,
|
|
"ht_mnpo/logit": 0.17855484783649445,
|
|
"ht_mnpo/residual": 0.17105485498905182,
|
|
"ht_mnpo/residual_abs": 0.18604342639446259,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.2527213096618652,
|
|
"logits/rejected": -2.2769358158111572,
|
|
"logps/chosen": -0.3151988387107849,
|
|
"logps/rejected": -0.2924613654613495,
|
|
"loss": 0.877219557762146,
|
|
"loss/core": 0.877219557762146,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.44504451751709,
|
|
"rewards/margins": 1.785548448562622,
|
|
"rewards/rejected": 0.6594961881637573,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.7205917442137915,
|
|
"grad_norm": 56.0,
|
|
"ht_mnpo/logit": 0.170209139585495,
|
|
"ht_mnpo/residual": 0.16270914673805237,
|
|
"ht_mnpo/residual_abs": 0.17232632637023926,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -2.1373720169067383,
|
|
"logits/rejected": -2.106365203857422,
|
|
"logps/chosen": -0.28223562240600586,
|
|
"logps/rejected": -0.27106139063835144,
|
|
"loss": 0.3418263792991638,
|
|
"loss/core": 0.3418263792991638,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8122715950012207,
|
|
"rewards/margins": 1.7020914554595947,
|
|
"rewards/rejected": 1.1101800203323364,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7253638749701742,
|
|
"grad_norm": 7.03125,
|
|
"ht_mnpo/logit": 0.17837955057621002,
|
|
"ht_mnpo/residual": 0.1708795577287674,
|
|
"ht_mnpo/residual_abs": 0.22366230189800262,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -2.1923656463623047,
|
|
"logits/rejected": -2.134833812713623,
|
|
"logps/chosen": -0.2628360092639923,
|
|
"logps/rejected": -0.2624150216579437,
|
|
"loss": 0.4018256664276123,
|
|
"loss/core": 0.4018256664276123,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.6742782592773438,
|
|
"rewards/margins": 1.7837955951690674,
|
|
"rewards/rejected": 1.8904825448989868,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7301360057265569,
|
|
"grad_norm": 19.5,
|
|
"ht_mnpo/logit": 0.07248622924089432,
|
|
"ht_mnpo/residual": 0.0649862214922905,
|
|
"ht_mnpo/residual_abs": 0.1020934134721756,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -2.1482584476470947,
|
|
"logits/rejected": -2.178565263748169,
|
|
"logps/chosen": -0.28691160678863525,
|
|
"logps/rejected": -0.29412776231765747,
|
|
"loss": 0.08699820935726166,
|
|
"loss/core": 0.08699820935726166,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.3631985187530518,
|
|
"rewards/margins": 0.7248622179031372,
|
|
"rewards/rejected": 0.6383363604545593,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7349081364829396,
|
|
"grad_norm": 74.0,
|
|
"ht_mnpo/logit": 0.04947735741734505,
|
|
"ht_mnpo/residual": 0.04197736084461212,
|
|
"ht_mnpo/residual_abs": 0.23659929633140564,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -2.1913866996765137,
|
|
"logits/rejected": -2.0948822498321533,
|
|
"logps/chosen": -0.27521374821662903,
|
|
"logps/rejected": -0.2676638662815094,
|
|
"loss": 0.31541621685028076,
|
|
"loss/core": 0.31541621685028076,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8486695289611816,
|
|
"rewards/margins": 0.4947734475135803,
|
|
"rewards/rejected": 2.353895902633667,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7396802672393223,
|
|
"grad_norm": 2.734375,
|
|
"ht_mnpo/logit": 0.1934659779071808,
|
|
"ht_mnpo/residual": 0.18596597015857697,
|
|
"ht_mnpo/residual_abs": 0.21423058211803436,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -2.0273308753967285,
|
|
"logits/rejected": -2.0538253784179688,
|
|
"logps/chosen": -0.3183925747871399,
|
|
"logps/rejected": -0.29416340589523315,
|
|
"loss": 0.6891728639602661,
|
|
"loss/core": 0.6891728639602661,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.740612030029297,
|
|
"rewards/margins": 1.934659719467163,
|
|
"rewards/rejected": 0.8059521913528442,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7444523979957051,
|
|
"grad_norm": 1.25,
|
|
"ht_mnpo/logit": 0.08660490810871124,
|
|
"ht_mnpo/residual": 0.07910490036010742,
|
|
"ht_mnpo/residual_abs": 0.08363465964794159,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.064676284790039,
|
|
"logits/rejected": -2.143580198287964,
|
|
"logps/chosen": -0.2868286669254303,
|
|
"logps/rejected": -0.26787668466567993,
|
|
"loss": 0.03424781560897827,
|
|
"loss/core": 0.03424781560897827,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0800483226776123,
|
|
"rewards/margins": 0.8660489916801453,
|
|
"rewards/rejected": 1.2139991521835327,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7492245287520878,
|
|
"grad_norm": 436.0,
|
|
"ht_mnpo/logit": 0.14060871303081512,
|
|
"ht_mnpo/residual": 0.1331087350845337,
|
|
"ht_mnpo/residual_abs": 0.1878836750984192,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -2.108158588409424,
|
|
"logits/rejected": -2.0105137825012207,
|
|
"logps/chosen": -0.28803449869155884,
|
|
"logps/rejected": -0.3160260319709778,
|
|
"loss": 0.2309446781873703,
|
|
"loss/core": 0.2309446781873703,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6357979774475098,
|
|
"rewards/margins": 1.4060871601104736,
|
|
"rewards/rejected": 1.2297108173370361,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7539966595084705,
|
|
"grad_norm": 14.375,
|
|
"ht_mnpo/logit": 0.12588101625442505,
|
|
"ht_mnpo/residual": 0.11838103830814362,
|
|
"ht_mnpo/residual_abs": 0.13901808857917786,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -1.92262864112854,
|
|
"logits/rejected": -1.9975627660751343,
|
|
"logps/chosen": -0.30886396765708923,
|
|
"logps/rejected": -0.2946474552154541,
|
|
"loss": 0.2501571774482727,
|
|
"loss/core": 0.2501571774482727,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.383531093597412,
|
|
"rewards/margins": 1.25881028175354,
|
|
"rewards/rejected": 1.1247209310531616,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7587687902648532,
|
|
"grad_norm": 153.0,
|
|
"ht_mnpo/logit": 0.20453031361103058,
|
|
"ht_mnpo/residual": 0.19703029096126556,
|
|
"ht_mnpo/residual_abs": 0.2428302764892578,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.9690449237823486,
|
|
"logits/rejected": -1.9464256763458252,
|
|
"logps/chosen": -0.3275276720523834,
|
|
"logps/rejected": -0.32991674542427063,
|
|
"loss": 0.47860488295555115,
|
|
"loss/core": 0.47860488295555115,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 4.1561384201049805,
|
|
"rewards/margins": 2.0453028678894043,
|
|
"rewards/rejected": 2.110835313796997,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7635409210212359,
|
|
"grad_norm": 62.5,
|
|
"ht_mnpo/logit": 0.048203855752944946,
|
|
"ht_mnpo/residual": 0.040703851729631424,
|
|
"ht_mnpo/residual_abs": 0.19021469354629517,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.151543617248535,
|
|
"logits/rejected": -2.2043967247009277,
|
|
"logps/chosen": -0.2940928339958191,
|
|
"logps/rejected": -0.2848767042160034,
|
|
"loss": 0.5195012092590332,
|
|
"loss/core": 0.5195012092590332,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.649043560028076,
|
|
"rewards/margins": 0.4820384383201599,
|
|
"rewards/rejected": 2.1670050621032715,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7683130517776187,
|
|
"grad_norm": 66.0,
|
|
"ht_mnpo/logit": 0.1901978701353073,
|
|
"ht_mnpo/residual": 0.18269787728786469,
|
|
"ht_mnpo/residual_abs": 0.213016077876091,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.0276436805725098,
|
|
"logits/rejected": -2.1004996299743652,
|
|
"logps/chosen": -0.25963181257247925,
|
|
"logps/rejected": -0.2606484591960907,
|
|
"loss": 0.2585400938987732,
|
|
"loss/core": 0.2585400938987732,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2736213207244873,
|
|
"rewards/margins": 1.9019787311553955,
|
|
"rewards/rejected": 1.3716427087783813,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.7730851825340014,
|
|
"grad_norm": 7.46875,
|
|
"ht_mnpo/logit": 0.10098608583211899,
|
|
"ht_mnpo/residual": 0.09348608553409576,
|
|
"ht_mnpo/residual_abs": 0.09852565079927444,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.122680187225342,
|
|
"logits/rejected": -2.235227108001709,
|
|
"logps/chosen": -0.2972281575202942,
|
|
"logps/rejected": -0.2812325060367584,
|
|
"loss": 0.060658883303403854,
|
|
"loss/core": 0.060658883303403854,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0125930309295654,
|
|
"rewards/margins": 1.0098607540130615,
|
|
"rewards/rejected": 1.0027321577072144,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7778573132903842,
|
|
"grad_norm": 12.0,
|
|
"ht_mnpo/logit": 0.1887650042772293,
|
|
"ht_mnpo/residual": 0.1812649965286255,
|
|
"ht_mnpo/residual_abs": 0.18580570816993713,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -2.242436170578003,
|
|
"logits/rejected": -2.2910566329956055,
|
|
"logps/chosen": -0.2807729244232178,
|
|
"logps/rejected": -0.2695391774177551,
|
|
"loss": 0.46180328726768494,
|
|
"loss/core": 0.46180328726768494,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.7984201908111572,
|
|
"rewards/margins": 1.8876501321792603,
|
|
"rewards/rejected": 0.9107701182365417,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7826294440467669,
|
|
"grad_norm": 0.73828125,
|
|
"ht_mnpo/logit": 0.25789007544517517,
|
|
"ht_mnpo/residual": 0.25039008259773254,
|
|
"ht_mnpo/residual_abs": 0.25934451818466187,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -2.1225481033325195,
|
|
"logits/rejected": -2.116036891937256,
|
|
"logps/chosen": -0.29405006766319275,
|
|
"logps/rejected": -0.3059104084968567,
|
|
"loss": 0.9970185160636902,
|
|
"loss/core": 0.9970185160636902,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.760523557662964,
|
|
"rewards/margins": 2.5789003372192383,
|
|
"rewards/rejected": 1.1816229820251465,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.7874015748031497,
|
|
"grad_norm": 30.25,
|
|
"ht_mnpo/logit": 0.12419946491718292,
|
|
"ht_mnpo/residual": 0.1166994571685791,
|
|
"ht_mnpo/residual_abs": 0.17805686593055725,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.0180411338806152,
|
|
"logits/rejected": -2.0512144565582275,
|
|
"logps/chosen": -0.2761440873146057,
|
|
"logps/rejected": -0.27113255858421326,
|
|
"loss": 0.39623942971229553,
|
|
"loss/core": 0.39623942971229553,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.456834554672241,
|
|
"rewards/margins": 1.2419946193695068,
|
|
"rewards/rejected": 1.2148399353027344,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7921737055595324,
|
|
"grad_norm": 280.0,
|
|
"ht_mnpo/logit": 0.1404004991054535,
|
|
"ht_mnpo/residual": 0.13290049135684967,
|
|
"ht_mnpo/residual_abs": 0.209401935338974,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.013359785079956,
|
|
"logits/rejected": -2.045358657836914,
|
|
"logps/chosen": -0.2946194112300873,
|
|
"logps/rejected": -0.2898050844669342,
|
|
"loss": 0.4411865770816803,
|
|
"loss/core": 0.4411865770816803,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.967160940170288,
|
|
"rewards/margins": 1.4040049314498901,
|
|
"rewards/rejected": 1.5631558895111084,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7969458363159151,
|
|
"grad_norm": 7.65625,
|
|
"ht_mnpo/logit": 0.1146935448050499,
|
|
"ht_mnpo/residual": 0.10719354450702667,
|
|
"ht_mnpo/residual_abs": 0.1680370271205902,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -1.9626811742782593,
|
|
"logits/rejected": -1.9111385345458984,
|
|
"logps/chosen": -0.3240414261817932,
|
|
"logps/rejected": -0.27530187368392944,
|
|
"loss": 0.43495216965675354,
|
|
"loss/core": 0.43495216965675354,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.719301700592041,
|
|
"rewards/margins": 1.1469354629516602,
|
|
"rewards/rejected": 1.5723663568496704,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8017179670722978,
|
|
"grad_norm": 720.0,
|
|
"ht_mnpo/logit": -0.0722452700138092,
|
|
"ht_mnpo/residual": -0.07974526286125183,
|
|
"ht_mnpo/residual_abs": 0.19856789708137512,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.2160840034484863,
|
|
"logits/rejected": -2.060520648956299,
|
|
"logps/chosen": -0.28456607460975647,
|
|
"logps/rejected": -0.29475289583206177,
|
|
"loss": 0.600700318813324,
|
|
"loss/core": 0.600700318813324,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7836272716522217,
|
|
"rewards/margins": -0.7224526405334473,
|
|
"rewards/rejected": 2.506080150604248,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8064900978286805,
|
|
"grad_norm": 3.734375,
|
|
"ht_mnpo/logit": 0.057844363152980804,
|
|
"ht_mnpo/residual": 0.05034436658024788,
|
|
"ht_mnpo/residual_abs": 0.18020616471767426,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -2.0028698444366455,
|
|
"logits/rejected": -2.0154130458831787,
|
|
"logps/chosen": -0.3073137104511261,
|
|
"logps/rejected": -0.3115970492362976,
|
|
"loss": 0.3085273802280426,
|
|
"loss/core": 0.3085273802280426,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.327054500579834,
|
|
"rewards/margins": 0.5784436464309692,
|
|
"rewards/rejected": 1.7486108541488647,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8112622285850632,
|
|
"grad_norm": 1.3671875,
|
|
"ht_mnpo/logit": 0.1011844128370285,
|
|
"ht_mnpo/residual": 0.09368441998958588,
|
|
"ht_mnpo/residual_abs": 0.09876108914613724,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -2.039194345474243,
|
|
"logits/rejected": -2.047559976577759,
|
|
"logps/chosen": -0.28829044103622437,
|
|
"logps/rejected": -0.2785061001777649,
|
|
"loss": 0.08063577860593796,
|
|
"loss/core": 0.08063577860593796,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0972280502319336,
|
|
"rewards/margins": 1.0118441581726074,
|
|
"rewards/rejected": 1.0853837728500366,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.816034359341446,
|
|
"grad_norm": 1.0234375,
|
|
"ht_mnpo/logit": 0.057864684611558914,
|
|
"ht_mnpo/residual": 0.050364673137664795,
|
|
"ht_mnpo/residual_abs": 0.07226506620645523,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -2.10422945022583,
|
|
"logits/rejected": -2.046278476715088,
|
|
"logps/chosen": -0.2990795373916626,
|
|
"logps/rejected": -0.28812864422798157,
|
|
"loss": 0.02833705022931099,
|
|
"loss/core": 0.02833705022931099,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.6183888912200928,
|
|
"rewards/margins": 0.5786467790603638,
|
|
"rewards/rejected": 1.039742112159729,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8208064900978287,
|
|
"grad_norm": 16.5,
|
|
"ht_mnpo/logit": 0.15453629195690155,
|
|
"ht_mnpo/residual": 0.14703628420829773,
|
|
"ht_mnpo/residual_abs": 0.15095371007919312,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -2.239527702331543,
|
|
"logits/rejected": -2.4272561073303223,
|
|
"logps/chosen": -0.2691555619239807,
|
|
"logps/rejected": -0.25069355964660645,
|
|
"loss": 0.25770825147628784,
|
|
"loss/core": 0.25770825147628784,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0894973278045654,
|
|
"rewards/margins": 1.5453628301620483,
|
|
"rewards/rejected": 1.5441343784332275,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.8255786208542114,
|
|
"grad_norm": 1.9140625,
|
|
"ht_mnpo/logit": 0.08422042429447174,
|
|
"ht_mnpo/residual": 0.07672043144702911,
|
|
"ht_mnpo/residual_abs": 0.10378699004650116,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -2.023658275604248,
|
|
"logits/rejected": -2.0356650352478027,
|
|
"logps/chosen": -0.3199964165687561,
|
|
"logps/rejected": -0.2985342741012573,
|
|
"loss": 0.25588542222976685,
|
|
"loss/core": 0.25588542222976685,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.785983681678772,
|
|
"rewards/margins": 0.8422042727470398,
|
|
"rewards/rejected": 0.9437793493270874,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8303507516105941,
|
|
"grad_norm": 932.0,
|
|
"ht_mnpo/logit": 0.014267271384596825,
|
|
"ht_mnpo/residual": 0.006767272017896175,
|
|
"ht_mnpo/residual_abs": 0.1964888572692871,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.158407688140869,
|
|
"logits/rejected": -2.176384449005127,
|
|
"logps/chosen": -0.24673886597156525,
|
|
"logps/rejected": -0.25381556153297424,
|
|
"loss": 0.5949617624282837,
|
|
"loss/core": 0.5949617624282837,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.17032790184021,
|
|
"rewards/margins": 0.14267274737358093,
|
|
"rewards/rejected": 2.0276553630828857,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8351228823669768,
|
|
"grad_norm": 1672.0,
|
|
"ht_mnpo/logit": 0.18652717769145966,
|
|
"ht_mnpo/residual": 0.17902716994285583,
|
|
"ht_mnpo/residual_abs": 0.22813236713409424,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -2.046302080154419,
|
|
"logits/rejected": -2.0166726112365723,
|
|
"logps/chosen": -0.2716887295246124,
|
|
"logps/rejected": -0.29062873125076294,
|
|
"loss": 0.7612979412078857,
|
|
"loss/core": 0.7612979412078857,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7709009647369385,
|
|
"rewards/margins": 1.8652719259262085,
|
|
"rewards/rejected": 0.9056289792060852,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.8398950131233596,
|
|
"grad_norm": 2.5,
|
|
"ht_mnpo/logit": 0.04809325188398361,
|
|
"ht_mnpo/residual": 0.04059325158596039,
|
|
"ht_mnpo/residual_abs": 0.04977937787771225,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.058687686920166,
|
|
"logits/rejected": -2.1479878425598145,
|
|
"logps/chosen": -0.2778651714324951,
|
|
"logps/rejected": -0.2754516899585724,
|
|
"loss": 0.008843088522553444,
|
|
"loss/core": 0.008843088522553444,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.213937520980835,
|
|
"rewards/margins": 0.4809325337409973,
|
|
"rewards/rejected": 0.7330051064491272,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8446671438797423,
|
|
"grad_norm": 1.9765625,
|
|
"ht_mnpo/logit": 0.07707710564136505,
|
|
"ht_mnpo/residual": 0.06957711279392242,
|
|
"ht_mnpo/residual_abs": 0.08699636161327362,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -2.0416557788848877,
|
|
"logits/rejected": -2.0932555198669434,
|
|
"logps/chosen": -0.3003470003604889,
|
|
"logps/rejected": -0.28412142395973206,
|
|
"loss": 0.04094807058572769,
|
|
"loss/core": 0.04094807058572769,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.4210716485977173,
|
|
"rewards/margins": 0.7707711458206177,
|
|
"rewards/rejected": 0.6503006219863892,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.849439274636125,
|
|
"grad_norm": 648.0,
|
|
"ht_mnpo/logit": 0.17258305847644806,
|
|
"ht_mnpo/residual": 0.16508305072784424,
|
|
"ht_mnpo/residual_abs": 0.2527818977832794,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -2.100677967071533,
|
|
"logits/rejected": -2.1688926219940186,
|
|
"logps/chosen": -0.29442304372787476,
|
|
"logps/rejected": -0.33285215497016907,
|
|
"loss": 0.8560763597488403,
|
|
"loss/core": 0.8560763597488403,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4236044883728027,
|
|
"rewards/margins": 1.7258304357528687,
|
|
"rewards/rejected": 1.6977739334106445,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.8542114053925077,
|
|
"grad_norm": 16.625,
|
|
"ht_mnpo/logit": 0.14334656298160553,
|
|
"ht_mnpo/residual": 0.1358465701341629,
|
|
"ht_mnpo/residual_abs": 0.14593496918678284,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -2.189239501953125,
|
|
"logits/rejected": -2.1437931060791016,
|
|
"logps/chosen": -0.30115875601768494,
|
|
"logps/rejected": -0.27261587977409363,
|
|
"loss": 0.3960161805152893,
|
|
"loss/core": 0.3960161805152893,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.458888530731201,
|
|
"rewards/margins": 1.4334657192230225,
|
|
"rewards/rejected": 1.0254226922988892,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8589835361488904,
|
|
"grad_norm": 0.265625,
|
|
"ht_mnpo/logit": 0.10469138622283936,
|
|
"ht_mnpo/residual": 0.09719139337539673,
|
|
"ht_mnpo/residual_abs": 0.10287211835384369,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -2.1673264503479004,
|
|
"logits/rejected": -2.2037110328674316,
|
|
"logps/chosen": -0.3311048448085785,
|
|
"logps/rejected": -0.31003594398498535,
|
|
"loss": 0.20032985508441925,
|
|
"loss/core": 0.20032985508441925,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4984869956970215,
|
|
"rewards/margins": 1.0469138622283936,
|
|
"rewards/rejected": 1.451573133468628,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8589835361488904,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.3534543008564247,
|
|
"train_runtime": 7070.5487,
|
|
"train_samples_per_second": 2.037,
|
|
"train_steps_per_second": 0.127
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|