c1dfc4f2fa
Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s43 Source: Original Platform
3824 lines
143 KiB
JSON
3824 lines
143 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8600609209819029,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004778116227677238,
|
|
"grad_norm": 250.0,
|
|
"ht_mnpo/logit": 0.07157169282436371,
|
|
"ht_mnpo/residual": 0.06407169252634048,
|
|
"ht_mnpo/residual_abs": 0.18986251950263977,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -2.270185708999634,
|
|
"logits/rejected": -1.9820005893707275,
|
|
"logps/chosen": -0.26669391989707947,
|
|
"logps/rejected": -0.28120771050453186,
|
|
"loss": 0.27043819427490234,
|
|
"loss/core": 0.27043819427490234,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.370647430419922,
|
|
"rewards/margins": 0.7157168388366699,
|
|
"rewards/rejected": 1.6549304723739624,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009556232455354476,
|
|
"grad_norm": 156.0,
|
|
"ht_mnpo/logit": 0.02802388370037079,
|
|
"ht_mnpo/residual": 0.020523879677057266,
|
|
"ht_mnpo/residual_abs": 0.1323598325252533,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.2279629707336426,
|
|
"logits/rejected": -1.9304723739624023,
|
|
"logps/chosen": -0.3020592927932739,
|
|
"logps/rejected": -0.304666131734848,
|
|
"loss": 0.18070298433303833,
|
|
"loss/core": 0.18070298433303833,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.5490913391113281,
|
|
"rewards/margins": 0.2802388072013855,
|
|
"rewards/rejected": 1.2688524723052979,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014334348683031715,
|
|
"grad_norm": 22.375,
|
|
"ht_mnpo/logit": 0.052930913865566254,
|
|
"ht_mnpo/residual": 0.04543091356754303,
|
|
"ht_mnpo/residual_abs": 0.23316678404808044,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.215111255645752,
|
|
"logits/rejected": -1.8859649896621704,
|
|
"logps/chosen": -0.2659865915775299,
|
|
"logps/rejected": -0.2871326804161072,
|
|
"loss": 0.708215057849884,
|
|
"loss/core": 0.708215057849884,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4834492206573486,
|
|
"rewards/margins": 0.529309093952179,
|
|
"rewards/rejected": 1.954140067100525,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.019112464910708952,
|
|
"grad_norm": 10.375,
|
|
"ht_mnpo/logit": 0.099542997777462,
|
|
"ht_mnpo/residual": 0.09204299747943878,
|
|
"ht_mnpo/residual_abs": 0.11109743267297745,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -2.171400785446167,
|
|
"logits/rejected": -2.030099630355835,
|
|
"logps/chosen": -0.27144134044647217,
|
|
"logps/rejected": -0.28609132766723633,
|
|
"loss": 0.08086283504962921,
|
|
"loss/core": 0.08086283504962921,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.783445119857788,
|
|
"rewards/margins": 0.9954298734664917,
|
|
"rewards/rejected": 0.7880153656005859,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.023890581138386192,
|
|
"grad_norm": 1.96875,
|
|
"ht_mnpo/logit": 0.12193785607814789,
|
|
"ht_mnpo/residual": 0.11443785578012466,
|
|
"ht_mnpo/residual_abs": 0.19381213188171387,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.0908684730529785,
|
|
"logits/rejected": -1.8903487920761108,
|
|
"logps/chosen": -0.27843743562698364,
|
|
"logps/rejected": -0.2850627899169922,
|
|
"loss": 0.640637218952179,
|
|
"loss/core": 0.640637218952179,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.377429485321045,
|
|
"rewards/margins": 1.2193785905838013,
|
|
"rewards/rejected": 1.158050775527954,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.02866869736606343,
|
|
"grad_norm": 110.5,
|
|
"ht_mnpo/logit": 0.003302433993667364,
|
|
"ht_mnpo/residual": -0.004197552800178528,
|
|
"ht_mnpo/residual_abs": 0.19736668467521667,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.2374136447906494,
|
|
"logits/rejected": -2.1604771614074707,
|
|
"logps/chosen": -0.2896232604980469,
|
|
"logps/rejected": -0.2754831314086914,
|
|
"loss": 0.5999057292938232,
|
|
"loss/core": 0.5999057292938232,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.1399874687194824,
|
|
"rewards/margins": 0.03302428871393204,
|
|
"rewards/rejected": 2.1069629192352295,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.03344681359374067,
|
|
"grad_norm": 0.51953125,
|
|
"ht_mnpo/logit": 0.14462466537952423,
|
|
"ht_mnpo/residual": 0.1371246576309204,
|
|
"ht_mnpo/residual_abs": 0.15740394592285156,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -2.2604546546936035,
|
|
"logits/rejected": -2.0551838874816895,
|
|
"logps/chosen": -0.2980991303920746,
|
|
"logps/rejected": -0.2945564389228821,
|
|
"loss": 0.8268828392028809,
|
|
"loss/core": 0.8268828392028809,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3615596294403076,
|
|
"rewards/margins": 1.44624662399292,
|
|
"rewards/rejected": 0.9153128862380981,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.038224929821417904,
|
|
"grad_norm": 1.0078125,
|
|
"ht_mnpo/logit": 0.06618395447731018,
|
|
"ht_mnpo/residual": 0.05868394300341606,
|
|
"ht_mnpo/residual_abs": 0.1978856474161148,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.577958822250366,
|
|
"logits/rejected": -2.3133766651153564,
|
|
"logps/chosen": -0.2794923782348633,
|
|
"logps/rejected": -0.2849942445755005,
|
|
"loss": 0.6078843474388123,
|
|
"loss/core": 0.6078843474388123,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4120702743530273,
|
|
"rewards/margins": 0.6618391275405884,
|
|
"rewards/rejected": 1.7502310276031494,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04300304604909514,
|
|
"grad_norm": 47.25,
|
|
"ht_mnpo/logit": 0.02121608331799507,
|
|
"ht_mnpo/residual": 0.013716082088649273,
|
|
"ht_mnpo/residual_abs": 0.14117369055747986,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -2.0499496459960938,
|
|
"logits/rejected": -1.7805042266845703,
|
|
"logps/chosen": -0.26364803314208984,
|
|
"logps/rejected": -0.2629775404930115,
|
|
"loss": 0.11064072698354721,
|
|
"loss/core": 0.11064072698354721,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 2.472045421600342,
|
|
"rewards/margins": 0.21216082572937012,
|
|
"rewards/rejected": 2.259884834289551,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.047781162276772385,
|
|
"grad_norm": 161.0,
|
|
"ht_mnpo/logit": 0.08355962485074997,
|
|
"ht_mnpo/residual": 0.07605963945388794,
|
|
"ht_mnpo/residual_abs": 0.1379310041666031,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -2.0790011882781982,
|
|
"logits/rejected": -1.9761953353881836,
|
|
"logps/chosen": -0.2877221703529358,
|
|
"logps/rejected": -0.29205089807510376,
|
|
"loss": 0.10855451971292496,
|
|
"loss/core": 0.10855451971292496,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.1596500873565674,
|
|
"rewards/margins": 0.8355963826179504,
|
|
"rewards/rejected": 1.3240535259246826,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05255927850444962,
|
|
"grad_norm": 0.4609375,
|
|
"ht_mnpo/logit": 0.044849641621112823,
|
|
"ht_mnpo/residual": 0.037349633872509,
|
|
"ht_mnpo/residual_abs": 0.13990451395511627,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -2.390953302383423,
|
|
"logits/rejected": -2.1549553871154785,
|
|
"logps/chosen": -0.2689022123813629,
|
|
"logps/rejected": -0.2913646996021271,
|
|
"loss": 0.48172539472579956,
|
|
"loss/core": 0.48172539472579956,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.500528335571289,
|
|
"rewards/margins": 0.44849643111228943,
|
|
"rewards/rejected": 1.0520318746566772,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.05733739473212686,
|
|
"grad_norm": 2.609375,
|
|
"ht_mnpo/logit": 0.05682535097002983,
|
|
"ht_mnpo/residual": 0.04932534694671631,
|
|
"ht_mnpo/residual_abs": 0.09572498500347137,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -2.295361042022705,
|
|
"logits/rejected": -2.1259632110595703,
|
|
"logps/chosen": -0.2934691905975342,
|
|
"logps/rejected": -0.2943810522556305,
|
|
"loss": 0.0800308957695961,
|
|
"loss/core": 0.0800308957695961,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6094799041748047,
|
|
"rewards/margins": 0.5682535171508789,
|
|
"rewards/rejected": 1.0412262678146362,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.062115510959804096,
|
|
"grad_norm": 63.0,
|
|
"ht_mnpo/logit": 0.060111574828624725,
|
|
"ht_mnpo/residual": 0.0526115782558918,
|
|
"ht_mnpo/residual_abs": 0.09094792604446411,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -2.3533196449279785,
|
|
"logits/rejected": -2.0280025005340576,
|
|
"logps/chosen": -0.3026924431324005,
|
|
"logps/rejected": -0.3124319016933441,
|
|
"loss": 0.05209514498710632,
|
|
"loss/core": 0.05209514498710632,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.5579849481582642,
|
|
"rewards/margins": 0.6011158227920532,
|
|
"rewards/rejected": 0.9568690061569214,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06689362718748133,
|
|
"grad_norm": 22.625,
|
|
"ht_mnpo/logit": 0.0568360760807991,
|
|
"ht_mnpo/residual": 0.04933607205748558,
|
|
"ht_mnpo/residual_abs": 0.08929934352636337,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -2.2468817234039307,
|
|
"logits/rejected": -2.062488079071045,
|
|
"logps/chosen": -0.30542343854904175,
|
|
"logps/rejected": -0.3070066273212433,
|
|
"loss": 0.06507541984319687,
|
|
"loss/core": 0.06507541984319687,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.744479775428772,
|
|
"rewards/margins": 0.5683607459068298,
|
|
"rewards/rejected": 1.176119089126587,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07167174341515857,
|
|
"grad_norm": 27.5,
|
|
"ht_mnpo/logit": 0.09052614867687225,
|
|
"ht_mnpo/residual": 0.08302616328001022,
|
|
"ht_mnpo/residual_abs": 0.13352297246456146,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -2.002654552459717,
|
|
"logits/rejected": -1.8294117450714111,
|
|
"logps/chosen": -0.3003098964691162,
|
|
"logps/rejected": -0.3102906346321106,
|
|
"loss": 0.1903291791677475,
|
|
"loss/core": 0.1903291791677475,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.9985805749893188,
|
|
"rewards/margins": 0.9052616357803345,
|
|
"rewards/rejected": 1.0933191776275635,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07644985964283581,
|
|
"grad_norm": 165.0,
|
|
"ht_mnpo/logit": 0.0681958794593811,
|
|
"ht_mnpo/residual": 0.060695894062519073,
|
|
"ht_mnpo/residual_abs": 0.19254747033119202,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.140778064727783,
|
|
"logits/rejected": -1.9811999797821045,
|
|
"logps/chosen": -0.27189767360687256,
|
|
"logps/rejected": -0.2811708450317383,
|
|
"loss": 0.4638647437095642,
|
|
"loss/core": 0.4638647437095642,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.553395986557007,
|
|
"rewards/margins": 0.681958794593811,
|
|
"rewards/rejected": 1.871437430381775,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08122797587051304,
|
|
"grad_norm": 5.09375,
|
|
"ht_mnpo/logit": 0.041115902364254,
|
|
"ht_mnpo/residual": 0.033615902066230774,
|
|
"ht_mnpo/residual_abs": 0.096328504383564,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.344470977783203,
|
|
"logits/rejected": -2.041018009185791,
|
|
"logps/chosen": -0.2965394854545593,
|
|
"logps/rejected": -0.30132466554641724,
|
|
"loss": 0.10672229528427124,
|
|
"loss/core": 0.10672229528427124,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.5701828002929688,
|
|
"rewards/margins": 0.4111589789390564,
|
|
"rewards/rejected": 1.159023642539978,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08600609209819028,
|
|
"grad_norm": 95.0,
|
|
"ht_mnpo/logit": 0.013142749667167664,
|
|
"ht_mnpo/residual": 0.0056427521631121635,
|
|
"ht_mnpo/residual_abs": 0.12206294387578964,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.291677951812744,
|
|
"logits/rejected": -2.0287234783172607,
|
|
"logps/chosen": -0.25615039467811584,
|
|
"logps/rejected": -0.27058348059654236,
|
|
"loss": 0.11879338324069977,
|
|
"loss/core": 0.11879338324069977,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6907819509506226,
|
|
"rewards/margins": 0.1314275860786438,
|
|
"rewards/rejected": 1.559354543685913,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09078420832586753,
|
|
"grad_norm": 20.75,
|
|
"ht_mnpo/logit": -0.08061865717172623,
|
|
"ht_mnpo/residual": -0.08811865001916885,
|
|
"ht_mnpo/residual_abs": 0.2554932236671448,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.2245447635650635,
|
|
"logits/rejected": -1.850603699684143,
|
|
"logps/chosen": -0.28188449144363403,
|
|
"logps/rejected": -0.299966961145401,
|
|
"loss": 0.804580807685852,
|
|
"loss/core": 0.804580807685852,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.833380103111267,
|
|
"rewards/margins": -0.8061864972114563,
|
|
"rewards/rejected": 2.63956618309021,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09556232455354477,
|
|
"grad_norm": 40.25,
|
|
"ht_mnpo/logit": 0.028320733457803726,
|
|
"ht_mnpo/residual": 0.02082073874771595,
|
|
"ht_mnpo/residual_abs": 0.06879810988903046,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -2.1782548427581787,
|
|
"logits/rejected": -1.9620933532714844,
|
|
"logps/chosen": -0.27486157417297363,
|
|
"logps/rejected": -0.2764720916748047,
|
|
"loss": 0.039663899689912796,
|
|
"loss/core": 0.039663899689912796,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.7030102014541626,
|
|
"rewards/margins": 0.28320735692977905,
|
|
"rewards/rejected": 1.4198030233383179,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.100340440781222,
|
|
"grad_norm": 4.1875,
|
|
"ht_mnpo/logit": 0.04795417934656143,
|
|
"ht_mnpo/residual": 0.040454182773828506,
|
|
"ht_mnpo/residual_abs": 0.13497649133205414,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.1107373237609863,
|
|
"logits/rejected": -1.921032190322876,
|
|
"logps/chosen": -0.29457932710647583,
|
|
"logps/rejected": -0.2966824173927307,
|
|
"loss": 0.13080136477947235,
|
|
"loss/core": 0.13080136477947235,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.2252862453460693,
|
|
"rewards/margins": 0.4795418381690979,
|
|
"rewards/rejected": 1.7457443475723267,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10511855700889924,
|
|
"grad_norm": 12.375,
|
|
"ht_mnpo/logit": 0.17580091953277588,
|
|
"ht_mnpo/residual": 0.16830092668533325,
|
|
"ht_mnpo/residual_abs": 0.1905735582113266,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -2.214897871017456,
|
|
"logits/rejected": -1.9663807153701782,
|
|
"logps/chosen": -0.2909243702888489,
|
|
"logps/rejected": -0.29862135648727417,
|
|
"loss": 0.4717007577419281,
|
|
"loss/core": 0.4717007577419281,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.3238365650177,
|
|
"rewards/margins": 1.7580095529556274,
|
|
"rewards/rejected": 1.5658271312713623,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10989667323657648,
|
|
"grad_norm": 0.234375,
|
|
"ht_mnpo/logit": -0.0033284418750554323,
|
|
"ht_mnpo/residual": -0.010828444734215736,
|
|
"ht_mnpo/residual_abs": 0.11969165503978729,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -2.2799150943756104,
|
|
"logits/rejected": -2.022031307220459,
|
|
"logps/chosen": -0.31012484431266785,
|
|
"logps/rejected": -0.300957590341568,
|
|
"loss": 0.16961291432380676,
|
|
"loss/core": 0.16961291432380676,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.8041832447052002,
|
|
"rewards/margins": -0.03328438848257065,
|
|
"rewards/rejected": 1.8374675512313843,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11467478946425372,
|
|
"grad_norm": 7.875,
|
|
"ht_mnpo/logit": 0.037880975753068924,
|
|
"ht_mnpo/residual": 0.0303809754550457,
|
|
"ht_mnpo/residual_abs": 0.18625445663928986,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -2.1674695014953613,
|
|
"logits/rejected": -1.9356225728988647,
|
|
"logps/chosen": -0.2807902693748474,
|
|
"logps/rejected": -0.2767426073551178,
|
|
"loss": 0.4123569130897522,
|
|
"loss/core": 0.4123569130897522,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.6680102348327637,
|
|
"rewards/margins": 0.3788098692893982,
|
|
"rewards/rejected": 2.2892005443573,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11945290569193096,
|
|
"grad_norm": 0.458984375,
|
|
"ht_mnpo/logit": 0.02273702248930931,
|
|
"ht_mnpo/residual": 0.015237025916576385,
|
|
"ht_mnpo/residual_abs": 0.06529071182012558,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -2.3368637561798096,
|
|
"logits/rejected": -2.065802574157715,
|
|
"logps/chosen": -0.27813562750816345,
|
|
"logps/rejected": -0.30318138003349304,
|
|
"loss": 0.018811456859111786,
|
|
"loss/core": 0.018811456859111786,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.5549370050430298,
|
|
"rewards/margins": 0.2273702174425125,
|
|
"rewards/rejected": 1.3275667428970337,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.12423102191960819,
|
|
"grad_norm": 42.5,
|
|
"ht_mnpo/logit": 0.16898369789123535,
|
|
"ht_mnpo/residual": 0.16148369014263153,
|
|
"ht_mnpo/residual_abs": 0.18639250099658966,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -2.2218174934387207,
|
|
"logits/rejected": -2.0029990673065186,
|
|
"logps/chosen": -0.2937593460083008,
|
|
"logps/rejected": -0.2997642159461975,
|
|
"loss": 0.4487941861152649,
|
|
"loss/core": 0.4487941861152649,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.533940315246582,
|
|
"rewards/margins": 1.689836859703064,
|
|
"rewards/rejected": 0.8441034555435181,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12900913814728543,
|
|
"grad_norm": 31.75,
|
|
"ht_mnpo/logit": 0.07704837620258331,
|
|
"ht_mnpo/residual": 0.06954839080572128,
|
|
"ht_mnpo/residual_abs": 0.08229651302099228,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.335357189178467,
|
|
"logits/rejected": -2.066044807434082,
|
|
"logps/chosen": -0.269487202167511,
|
|
"logps/rejected": -0.273403525352478,
|
|
"loss": 0.03789084404706955,
|
|
"loss/core": 0.03789084404706955,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.7870798110961914,
|
|
"rewards/margins": 0.7704838514328003,
|
|
"rewards/rejected": 1.016595721244812,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13378725437496267,
|
|
"grad_norm": 3.734375,
|
|
"ht_mnpo/logit": 0.013967243023216724,
|
|
"ht_mnpo/residual": 0.006467245519161224,
|
|
"ht_mnpo/residual_abs": 0.06466107070446014,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.3525476455688477,
|
|
"logits/rejected": -2.1170451641082764,
|
|
"logps/chosen": -0.2825363278388977,
|
|
"logps/rejected": -0.2803972661495209,
|
|
"loss": 0.02001281827688217,
|
|
"loss/core": 0.02001281827688217,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.4781088829040527,
|
|
"rewards/margins": 0.1396724134683609,
|
|
"rewards/rejected": 1.338436484336853,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.1385653706026399,
|
|
"grad_norm": 0.216796875,
|
|
"ht_mnpo/logit": -0.0018466196488589048,
|
|
"ht_mnpo/residual": -0.009346622973680496,
|
|
"ht_mnpo/residual_abs": 0.09110254794359207,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -2.3927693367004395,
|
|
"logits/rejected": -2.182438611984253,
|
|
"logps/chosen": -0.2859313488006592,
|
|
"logps/rejected": -0.2825351059436798,
|
|
"loss": 0.17166972160339355,
|
|
"loss/core": 0.17166972160339355,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.1670573949813843,
|
|
"rewards/margins": -0.01846626028418541,
|
|
"rewards/rejected": 1.1855233907699585,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14334348683031714,
|
|
"grad_norm": 11.375,
|
|
"ht_mnpo/logit": 0.03970382362604141,
|
|
"ht_mnpo/residual": 0.032203830778598785,
|
|
"ht_mnpo/residual_abs": 0.2119670808315277,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.184814929962158,
|
|
"logits/rejected": -1.9867866039276123,
|
|
"logps/chosen": -0.27966034412384033,
|
|
"logps/rejected": -0.3121265172958374,
|
|
"loss": 0.901578426361084,
|
|
"loss/core": 0.901578426361084,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2797250747680664,
|
|
"rewards/margins": 0.3970384895801544,
|
|
"rewards/rejected": 1.8826866149902344,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.14812160305799438,
|
|
"grad_norm": 888.0,
|
|
"ht_mnpo/logit": 0.037043869495391846,
|
|
"ht_mnpo/residual": 0.029543865472078323,
|
|
"ht_mnpo/residual_abs": 0.2591553032398224,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.214869737625122,
|
|
"logits/rejected": -1.9647678136825562,
|
|
"logps/chosen": -0.29476529359817505,
|
|
"logps/rejected": -0.33797982335090637,
|
|
"loss": 0.9004504084587097,
|
|
"loss/core": 0.9004504084587097,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.5209193229675293,
|
|
"rewards/margins": 0.37043848633766174,
|
|
"rewards/rejected": 2.1504807472229004,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15289971928567161,
|
|
"grad_norm": 896.0,
|
|
"ht_mnpo/logit": 0.10632804781198502,
|
|
"ht_mnpo/residual": 0.09882805496454239,
|
|
"ht_mnpo/residual_abs": 0.17886283993721008,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.0985219478607178,
|
|
"logits/rejected": -1.8167247772216797,
|
|
"logps/chosen": -0.29428020119667053,
|
|
"logps/rejected": -0.30171844363212585,
|
|
"loss": 0.6274815797805786,
|
|
"loss/core": 0.6274815797805786,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.4871866703033447,
|
|
"rewards/margins": 1.0632803440093994,
|
|
"rewards/rejected": 1.4239060878753662,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15767783551334885,
|
|
"grad_norm": 2.296875,
|
|
"ht_mnpo/logit": 0.07954417169094086,
|
|
"ht_mnpo/residual": 0.07204417884349823,
|
|
"ht_mnpo/residual_abs": 0.07974160462617874,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.258405923843384,
|
|
"logits/rejected": -1.9701240062713623,
|
|
"logps/chosen": -0.2872469127178192,
|
|
"logps/rejected": -0.2988366186618805,
|
|
"loss": 0.11342660337686539,
|
|
"loss/core": 0.11342660337686539,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7711330652236938,
|
|
"rewards/margins": 0.7954418063163757,
|
|
"rewards/rejected": 0.9756911396980286,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.1624559517410261,
|
|
"grad_norm": 2.0625,
|
|
"ht_mnpo/logit": 0.14767876267433167,
|
|
"ht_mnpo/residual": 0.14017875492572784,
|
|
"ht_mnpo/residual_abs": 0.20500735938549042,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.192570447921753,
|
|
"logits/rejected": -2.0448708534240723,
|
|
"logps/chosen": -0.3334094285964966,
|
|
"logps/rejected": -0.3062793016433716,
|
|
"loss": 0.7441643476486206,
|
|
"loss/core": 0.7441643476486206,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.821702241897583,
|
|
"rewards/margins": 1.4767874479293823,
|
|
"rewards/rejected": 1.3449146747589111,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16723406796870333,
|
|
"grad_norm": 0.494140625,
|
|
"ht_mnpo/logit": 0.017073174938559532,
|
|
"ht_mnpo/residual": 0.009573178365826607,
|
|
"ht_mnpo/residual_abs": 0.11641023308038712,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.202766180038452,
|
|
"logits/rejected": -2.0135915279388428,
|
|
"logps/chosen": -0.27780964970588684,
|
|
"logps/rejected": -0.2846834063529968,
|
|
"loss": 0.12164388597011566,
|
|
"loss/core": 0.12164388597011566,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.587665319442749,
|
|
"rewards/margins": 0.17073175311088562,
|
|
"rewards/rejected": 1.4169334173202515,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17201218419638056,
|
|
"grad_norm": 288.0,
|
|
"ht_mnpo/logit": 0.1441071778535843,
|
|
"ht_mnpo/residual": 0.13660718500614166,
|
|
"ht_mnpo/residual_abs": 0.15255433320999146,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -2.3559622764587402,
|
|
"logits/rejected": -2.1174533367156982,
|
|
"logps/chosen": -0.2628288269042969,
|
|
"logps/rejected": -0.2772200405597687,
|
|
"loss": 0.320454478263855,
|
|
"loss/core": 0.320454478263855,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.7258729934692383,
|
|
"rewards/margins": 1.4410715103149414,
|
|
"rewards/rejected": 1.2848012447357178,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17679030042405783,
|
|
"grad_norm": 0.859375,
|
|
"ht_mnpo/logit": -0.04746574908494949,
|
|
"ht_mnpo/residual": -0.05496574565768242,
|
|
"ht_mnpo/residual_abs": 0.1622922122478485,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -2.2382771968841553,
|
|
"logits/rejected": -2.074378728866577,
|
|
"logps/chosen": -0.3004586696624756,
|
|
"logps/rejected": -0.2946397364139557,
|
|
"loss": 0.5814380049705505,
|
|
"loss/core": 0.5814380049705505,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.3076417446136475,
|
|
"rewards/margins": -0.4746573865413666,
|
|
"rewards/rejected": 1.782299280166626,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18156841665173507,
|
|
"grad_norm": 1008.0,
|
|
"ht_mnpo/logit": 0.15239554643630981,
|
|
"ht_mnpo/residual": 0.14489556849002838,
|
|
"ht_mnpo/residual_abs": 0.153962180018425,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -2.1816508769989014,
|
|
"logits/rejected": -2.0114400386810303,
|
|
"logps/chosen": -0.32795587182044983,
|
|
"logps/rejected": -0.29940658807754517,
|
|
"loss": 0.4533259868621826,
|
|
"loss/core": 0.4533259868621826,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.608320713043213,
|
|
"rewards/margins": 1.5239555835723877,
|
|
"rewards/rejected": 1.0843651294708252,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.1863465328794123,
|
|
"grad_norm": 11.8125,
|
|
"ht_mnpo/logit": 0.08571872860193253,
|
|
"ht_mnpo/residual": 0.0782187357544899,
|
|
"ht_mnpo/residual_abs": 0.17521581053733826,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -2.221156597137451,
|
|
"logits/rejected": -1.9798905849456787,
|
|
"logps/chosen": -0.3095605969429016,
|
|
"logps/rejected": -0.30117204785346985,
|
|
"loss": 0.31682509183883667,
|
|
"loss/core": 0.31682509183883667,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.334531545639038,
|
|
"rewards/margins": 0.8571873903274536,
|
|
"rewards/rejected": 1.477344274520874,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19112464910708954,
|
|
"grad_norm": 60.0,
|
|
"ht_mnpo/logit": 0.028831344097852707,
|
|
"ht_mnpo/residual": 0.021331345662474632,
|
|
"ht_mnpo/residual_abs": 0.08674871176481247,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -2.182504892349243,
|
|
"logits/rejected": -1.9949296712875366,
|
|
"logps/chosen": -0.31937891244888306,
|
|
"logps/rejected": -0.3028191924095154,
|
|
"loss": 0.05946077033877373,
|
|
"loss/core": 0.05946077033877373,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7149394750595093,
|
|
"rewards/margins": 0.2883134186267853,
|
|
"rewards/rejected": 1.4266260862350464,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19590276533476678,
|
|
"grad_norm": 17.0,
|
|
"ht_mnpo/logit": 0.08596853911876678,
|
|
"ht_mnpo/residual": 0.07846853882074356,
|
|
"ht_mnpo/residual_abs": 0.09083747863769531,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -2.3666977882385254,
|
|
"logits/rejected": -2.1427879333496094,
|
|
"logps/chosen": -0.2890446186065674,
|
|
"logps/rejected": -0.29884833097457886,
|
|
"loss": 0.06484078615903854,
|
|
"loss/core": 0.06484078615903854,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7943065166473389,
|
|
"rewards/margins": 0.8596854209899902,
|
|
"rewards/rejected": 0.9346210360527039,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.200680881562444,
|
|
"grad_norm": 4.96875,
|
|
"ht_mnpo/logit": 0.04586448892951012,
|
|
"ht_mnpo/residual": 0.03836449235677719,
|
|
"ht_mnpo/residual_abs": 0.04392882436513901,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -2.3530592918395996,
|
|
"logits/rejected": -2.1788575649261475,
|
|
"logps/chosen": -0.3264649212360382,
|
|
"logps/rejected": -0.32113516330718994,
|
|
"loss": 0.01702629216015339,
|
|
"loss/core": 0.01702629216015339,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 0.848310112953186,
|
|
"rewards/margins": 0.45864492654800415,
|
|
"rewards/rejected": 0.38966527581214905,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20545899779012125,
|
|
"grad_norm": 77.0,
|
|
"ht_mnpo/logit": 0.10470955073833466,
|
|
"ht_mnpo/residual": 0.09720954298973083,
|
|
"ht_mnpo/residual_abs": 0.1478584110736847,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.1095123291015625,
|
|
"logits/rejected": -1.9788084030151367,
|
|
"logps/chosen": -0.2908587157726288,
|
|
"logps/rejected": -0.30672210454940796,
|
|
"loss": 0.18497677147388458,
|
|
"loss/core": 0.18497677147388458,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.263723373413086,
|
|
"rewards/margins": 1.047095537185669,
|
|
"rewards/rejected": 1.216628074645996,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.2102371140177985,
|
|
"grad_norm": 236.0,
|
|
"ht_mnpo/logit": 0.021329589188098907,
|
|
"ht_mnpo/residual": 0.013829593546688557,
|
|
"ht_mnpo/residual_abs": 0.11300545930862427,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -2.06388521194458,
|
|
"logits/rejected": -1.8469206094741821,
|
|
"logps/chosen": -0.28565680980682373,
|
|
"logps/rejected": -0.29494622349739075,
|
|
"loss": 0.1155991330742836,
|
|
"loss/core": 0.1155991330742836,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.018338203430176,
|
|
"rewards/margins": 0.21329593658447266,
|
|
"rewards/rejected": 1.8050426244735718,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.21501523024547572,
|
|
"grad_norm": 10.0,
|
|
"ht_mnpo/logit": 0.1730707436800003,
|
|
"ht_mnpo/residual": 0.16557076573371887,
|
|
"ht_mnpo/residual_abs": 0.19626513123512268,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.2850561141967773,
|
|
"logits/rejected": -2.097651720046997,
|
|
"logps/chosen": -0.30766889452934265,
|
|
"logps/rejected": -0.30594539642333984,
|
|
"loss": 0.4711076617240906,
|
|
"loss/core": 0.4711076617240906,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 3.2686572074890137,
|
|
"rewards/margins": 1.7307075262069702,
|
|
"rewards/rejected": 1.537950038909912,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21979334647315296,
|
|
"grad_norm": 9.0,
|
|
"ht_mnpo/logit": 0.20456573367118835,
|
|
"ht_mnpo/residual": 0.19706572592258453,
|
|
"ht_mnpo/residual_abs": 0.23375244438648224,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -2.202606439590454,
|
|
"logits/rejected": -1.9548810720443726,
|
|
"logps/chosen": -0.30069398880004883,
|
|
"logps/rejected": -0.3219486176967621,
|
|
"loss": 0.7216225862503052,
|
|
"loss/core": 0.7216225862503052,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.3075389862060547,
|
|
"rewards/margins": 2.0456573963165283,
|
|
"rewards/rejected": 1.2618815898895264,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245714627008302,
|
|
"grad_norm": 206.0,
|
|
"ht_mnpo/logit": 0.031354598701000214,
|
|
"ht_mnpo/residual": 0.023854583501815796,
|
|
"ht_mnpo/residual_abs": 0.18463179469108582,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.1225569248199463,
|
|
"logits/rejected": -1.8874324560165405,
|
|
"logps/chosen": -0.2835076153278351,
|
|
"logps/rejected": -0.3018529713153839,
|
|
"loss": 0.449360191822052,
|
|
"loss/core": 0.449360191822052,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.9686769247055054,
|
|
"rewards/margins": 0.31354600191116333,
|
|
"rewards/rejected": 1.6551311016082764,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22934957892850744,
|
|
"grad_norm": 20.0,
|
|
"ht_mnpo/logit": 0.031544990837574005,
|
|
"ht_mnpo/residual": 0.024044986814260483,
|
|
"ht_mnpo/residual_abs": 0.1752614676952362,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.209033966064453,
|
|
"logits/rejected": -1.8773391246795654,
|
|
"logps/chosen": -0.27358725666999817,
|
|
"logps/rejected": -0.28609001636505127,
|
|
"loss": 0.25187164545059204,
|
|
"loss/core": 0.25187164545059204,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.8158626556396484,
|
|
"rewards/margins": 0.31544986367225647,
|
|
"rewards/rejected": 1.5004127025604248,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23412769515618467,
|
|
"grad_norm": 35.25,
|
|
"ht_mnpo/logit": 0.20634432137012482,
|
|
"ht_mnpo/residual": 0.1988442987203598,
|
|
"ht_mnpo/residual_abs": 0.24610598385334015,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.1333718299865723,
|
|
"logits/rejected": -1.875788688659668,
|
|
"logps/chosen": -0.29131993651390076,
|
|
"logps/rejected": -0.292595237493515,
|
|
"loss": 0.5379303693771362,
|
|
"loss/core": 0.5379303693771362,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.162245273590088,
|
|
"rewards/margins": 2.0634429454803467,
|
|
"rewards/rejected": 2.098802089691162,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.2389058113838619,
|
|
"grad_norm": 0.8984375,
|
|
"ht_mnpo/logit": 0.12799809873104095,
|
|
"ht_mnpo/residual": 0.12049808353185654,
|
|
"ht_mnpo/residual_abs": 0.17880940437316895,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -2.1053507328033447,
|
|
"logits/rejected": -1.8193076848983765,
|
|
"logps/chosen": -0.32429033517837524,
|
|
"logps/rejected": -0.29927438497543335,
|
|
"loss": 0.6025797724723816,
|
|
"loss/core": 0.6025797724723816,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.8428993225097656,
|
|
"rewards/margins": 1.2799808979034424,
|
|
"rewards/rejected": 1.5629183053970337,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24368392761153915,
|
|
"grad_norm": 4.875,
|
|
"ht_mnpo/logit": 0.13739672303199768,
|
|
"ht_mnpo/residual": 0.12989673018455505,
|
|
"ht_mnpo/residual_abs": 0.14480608701705933,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -2.117206573486328,
|
|
"logits/rejected": -1.9836727380752563,
|
|
"logps/chosen": -0.3060274124145508,
|
|
"logps/rejected": -0.30183497071266174,
|
|
"loss": 0.2743765711784363,
|
|
"loss/core": 0.2743765711784363,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.924288749694824,
|
|
"rewards/margins": 1.3739672899246216,
|
|
"rewards/rejected": 1.550321340560913,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.24846204383921638,
|
|
"grad_norm": 10.75,
|
|
"ht_mnpo/logit": 0.06477851420640945,
|
|
"ht_mnpo/residual": 0.05727851390838623,
|
|
"ht_mnpo/residual_abs": 0.07653465867042542,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -2.1527230739593506,
|
|
"logits/rejected": -1.9991321563720703,
|
|
"logps/chosen": -0.28355228900909424,
|
|
"logps/rejected": -0.2868475615978241,
|
|
"loss": 0.038742970675230026,
|
|
"loss/core": 0.038742970675230026,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.7354450225830078,
|
|
"rewards/margins": 0.6477851867675781,
|
|
"rewards/rejected": 1.0876598358154297,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25324016006689365,
|
|
"grad_norm": 60.5,
|
|
"ht_mnpo/logit": 0.0033052079379558563,
|
|
"ht_mnpo/residual": -0.004194788634777069,
|
|
"ht_mnpo/residual_abs": 0.09127648174762726,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.1906323432922363,
|
|
"logits/rejected": -1.9368772506713867,
|
|
"logps/chosen": -0.29127225279808044,
|
|
"logps/rejected": -0.28326812386512756,
|
|
"loss": 0.08905141055583954,
|
|
"loss/core": 0.08905141055583954,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.4620263576507568,
|
|
"rewards/margins": 0.03305206447839737,
|
|
"rewards/rejected": 1.4289741516113281,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.25801827629457086,
|
|
"grad_norm": 2.1875,
|
|
"ht_mnpo/logit": 0.043637726455926895,
|
|
"ht_mnpo/residual": 0.03613772615790367,
|
|
"ht_mnpo/residual_abs": 0.04942305386066437,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -2.236889123916626,
|
|
"logits/rejected": -2.070533514022827,
|
|
"logps/chosen": -0.30425819754600525,
|
|
"logps/rejected": -0.29968196153640747,
|
|
"loss": 0.012058206833899021,
|
|
"loss/core": 0.012058206833899021,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.2041269540786743,
|
|
"rewards/margins": 0.43637722730636597,
|
|
"rewards/rejected": 0.7677494883537292,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627963925222481,
|
|
"grad_norm": 1160.0,
|
|
"ht_mnpo/logit": 0.1649816781282425,
|
|
"ht_mnpo/residual": 0.15748167037963867,
|
|
"ht_mnpo/residual_abs": 0.21221141517162323,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.2023870944976807,
|
|
"logits/rejected": -2.0466129779815674,
|
|
"logps/chosen": -0.28603219985961914,
|
|
"logps/rejected": -0.27672260999679565,
|
|
"loss": 0.9627658724784851,
|
|
"loss/core": 0.9627658724784851,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.6714327335357666,
|
|
"rewards/margins": 1.6498167514801025,
|
|
"rewards/rejected": 1.021615982055664,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26757450874992533,
|
|
"grad_norm": 8.1875,
|
|
"ht_mnpo/logit": 0.008860750123858452,
|
|
"ht_mnpo/residual": 0.0013607516884803772,
|
|
"ht_mnpo/residual_abs": 0.15622496604919434,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.1357626914978027,
|
|
"logits/rejected": -1.8891808986663818,
|
|
"logps/chosen": -0.2642098069190979,
|
|
"logps/rejected": -0.28465694189071655,
|
|
"loss": 0.242923766374588,
|
|
"loss/core": 0.242923766374588,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.0400595664978027,
|
|
"rewards/margins": 0.08860747516155243,
|
|
"rewards/rejected": 1.9514522552490234,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723526249776026,
|
|
"grad_norm": 5.90625,
|
|
"ht_mnpo/logit": 0.11119908094406128,
|
|
"ht_mnpo/residual": 0.10369908809661865,
|
|
"ht_mnpo/residual_abs": 0.15644970536231995,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -1.981865644454956,
|
|
"logits/rejected": -1.7845958471298218,
|
|
"logps/chosen": -0.28549736738204956,
|
|
"logps/rejected": -0.27184969186782837,
|
|
"loss": 0.2947588562965393,
|
|
"loss/core": 0.2947588562965393,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6670725345611572,
|
|
"rewards/margins": 1.1119908094406128,
|
|
"rewards/rejected": 1.555082082748413,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.2771307412052798,
|
|
"grad_norm": 1.5546875,
|
|
"ht_mnpo/logit": 0.03820858150720596,
|
|
"ht_mnpo/residual": 0.030708584934473038,
|
|
"ht_mnpo/residual_abs": 0.058207251131534576,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -2.2808995246887207,
|
|
"logits/rejected": -2.0846009254455566,
|
|
"logps/chosen": -0.2757318615913391,
|
|
"logps/rejected": -0.2664252817630768,
|
|
"loss": 0.011909561231732368,
|
|
"loss/core": 0.011909561231732368,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.4301050901412964,
|
|
"rewards/margins": 0.3820858299732208,
|
|
"rewards/rejected": 1.0480191707611084,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28190885743295707,
|
|
"grad_norm": 744.0,
|
|
"ht_mnpo/logit": 0.15831990540027618,
|
|
"ht_mnpo/residual": 0.15081989765167236,
|
|
"ht_mnpo/residual_abs": 0.35883960127830505,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.3191518783569336,
|
|
"logits/rejected": -2.0893778800964355,
|
|
"logps/chosen": -0.31087130308151245,
|
|
"logps/rejected": -0.32478809356689453,
|
|
"loss": 1.9273204803466797,
|
|
"loss/core": 1.9273204803466797,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.648682117462158,
|
|
"rewards/margins": 1.5831987857818604,
|
|
"rewards/rejected": 2.0654828548431396,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.2866869736606343,
|
|
"grad_norm": 4.28125,
|
|
"ht_mnpo/logit": 0.0808088406920433,
|
|
"ht_mnpo/residual": 0.07330884784460068,
|
|
"ht_mnpo/residual_abs": 0.10061073303222656,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.9314266443252563,
|
|
"logits/rejected": -1.6769113540649414,
|
|
"logps/chosen": -0.3090340495109558,
|
|
"logps/rejected": -0.30936768651008606,
|
|
"loss": 0.0936809629201889,
|
|
"loss/core": 0.0936809629201889,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.8696715831756592,
|
|
"rewards/margins": 0.808088481426239,
|
|
"rewards/rejected": 1.0615832805633545,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.29146508988831155,
|
|
"grad_norm": 6.5,
|
|
"ht_mnpo/logit": 0.05852525308728218,
|
|
"ht_mnpo/residual": 0.05102524906396866,
|
|
"ht_mnpo/residual_abs": 0.139701247215271,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -2.129746675491333,
|
|
"logits/rejected": -1.9119545221328735,
|
|
"logps/chosen": -0.28492045402526855,
|
|
"logps/rejected": -0.29339781403541565,
|
|
"loss": 0.16279859840869904,
|
|
"loss/core": 0.16279859840869904,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.5068206787109375,
|
|
"rewards/margins": 0.5852525234222412,
|
|
"rewards/rejected": 1.9215682744979858,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.29624320611598876,
|
|
"grad_norm": 14.5,
|
|
"ht_mnpo/logit": 0.06949242204427719,
|
|
"ht_mnpo/residual": 0.06199241429567337,
|
|
"ht_mnpo/residual_abs": 0.0795050710439682,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -2.2989072799682617,
|
|
"logits/rejected": -2.007084369659424,
|
|
"logps/chosen": -0.28377217054367065,
|
|
"logps/rejected": -0.2830241024494171,
|
|
"loss": 0.0549379363656044,
|
|
"loss/core": 0.0549379363656044,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.7683111429214478,
|
|
"rewards/margins": 0.6949242353439331,
|
|
"rewards/rejected": 1.073386788368225,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.301021322343666,
|
|
"grad_norm": 6.4375,
|
|
"ht_mnpo/logit": 0.038666121661663055,
|
|
"ht_mnpo/residual": 0.03116612136363983,
|
|
"ht_mnpo/residual_abs": 0.04453511908650398,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -2.3783583641052246,
|
|
"logits/rejected": -2.1003165245056152,
|
|
"logps/chosen": -0.2959398627281189,
|
|
"logps/rejected": -0.3017369508743286,
|
|
"loss": 0.012863054871559143,
|
|
"loss/core": 0.012863054871559143,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 0.9655852317810059,
|
|
"rewards/margins": 0.38666123151779175,
|
|
"rewards/rejected": 0.5789240002632141,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.30579943857134323,
|
|
"grad_norm": 12.5625,
|
|
"ht_mnpo/logit": 0.009172516874969006,
|
|
"ht_mnpo/residual": 0.0016725212335586548,
|
|
"ht_mnpo/residual_abs": 0.1754622906446457,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -2.2958977222442627,
|
|
"logits/rejected": -2.032662868499756,
|
|
"logps/chosen": -0.2879457473754883,
|
|
"logps/rejected": -0.30056875944137573,
|
|
"loss": 0.37386396527290344,
|
|
"loss/core": 0.37386396527290344,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.9279342889785767,
|
|
"rewards/margins": 0.09172508865594864,
|
|
"rewards/rejected": 1.8362090587615967,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.3105775547990205,
|
|
"grad_norm": 11.5,
|
|
"ht_mnpo/logit": 0.21785497665405273,
|
|
"ht_mnpo/residual": 0.2103549689054489,
|
|
"ht_mnpo/residual_abs": 0.22608724236488342,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -2.0493011474609375,
|
|
"logits/rejected": -1.8347457647323608,
|
|
"logps/chosen": -0.2787044942378998,
|
|
"logps/rejected": -0.27662545442581177,
|
|
"loss": 0.6209802031517029,
|
|
"loss/core": 0.6209802031517029,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.5399444103240967,
|
|
"rewards/margins": 2.1785497665405273,
|
|
"rewards/rejected": 1.3613946437835693,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.3153556710266977,
|
|
"grad_norm": 19.125,
|
|
"ht_mnpo/logit": 0.14707127213478088,
|
|
"ht_mnpo/residual": 0.13957123458385468,
|
|
"ht_mnpo/residual_abs": 0.17394544184207916,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.2151148319244385,
|
|
"logits/rejected": -1.889974594116211,
|
|
"logps/chosen": -0.2892436385154724,
|
|
"logps/rejected": -0.2921144664287567,
|
|
"loss": 0.4615117907524109,
|
|
"loss/core": 0.4615117907524109,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.9435486793518066,
|
|
"rewards/margins": 1.470712423324585,
|
|
"rewards/rejected": 1.4728357791900635,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32013378725437497,
|
|
"grad_norm": 47.5,
|
|
"ht_mnpo/logit": 0.060556523501873016,
|
|
"ht_mnpo/residual": 0.05305652692914009,
|
|
"ht_mnpo/residual_abs": 0.1689012497663498,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.222839593887329,
|
|
"logits/rejected": -2.075805425643921,
|
|
"logps/chosen": -0.2583637237548828,
|
|
"logps/rejected": -0.2836921215057373,
|
|
"loss": 0.3947281241416931,
|
|
"loss/core": 0.3947281241416931,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.1558752059936523,
|
|
"rewards/margins": 0.6055654287338257,
|
|
"rewards/rejected": 1.550309658050537,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3249119034820522,
|
|
"grad_norm": 258.0,
|
|
"ht_mnpo/logit": 0.15883901715278625,
|
|
"ht_mnpo/residual": 0.15133902430534363,
|
|
"ht_mnpo/residual_abs": 0.16413579881191254,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -2.2033660411834717,
|
|
"logits/rejected": -1.9985862970352173,
|
|
"logps/chosen": -0.26738107204437256,
|
|
"logps/rejected": -0.2792484164237976,
|
|
"loss": 0.4059799313545227,
|
|
"loss/core": 0.4059799313545227,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.9665191173553467,
|
|
"rewards/margins": 1.5883904695510864,
|
|
"rewards/rejected": 1.3781286478042603,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.32969001970972944,
|
|
"grad_norm": 0.6875,
|
|
"ht_mnpo/logit": 0.15273532271385193,
|
|
"ht_mnpo/residual": 0.1452353298664093,
|
|
"ht_mnpo/residual_abs": 0.17430062592029572,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -2.4266574382781982,
|
|
"logits/rejected": -2.118107318878174,
|
|
"logps/chosen": -0.27640026807785034,
|
|
"logps/rejected": -0.28678444027900696,
|
|
"loss": 0.63597571849823,
|
|
"loss/core": 0.63597571849823,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5499401092529297,
|
|
"rewards/margins": 1.527353286743164,
|
|
"rewards/rejected": 1.0225868225097656,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33446813593740665,
|
|
"grad_norm": 7.84375,
|
|
"ht_mnpo/logit": 0.1263376772403717,
|
|
"ht_mnpo/residual": 0.11883767694234848,
|
|
"ht_mnpo/residual_abs": 0.14816422760486603,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.320239543914795,
|
|
"logits/rejected": -2.0900626182556152,
|
|
"logps/chosen": -0.2879655361175537,
|
|
"logps/rejected": -0.30228275060653687,
|
|
"loss": 0.41849765181541443,
|
|
"loss/core": 0.41849765181541443,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.399965286254883,
|
|
"rewards/margins": 1.2633768320083618,
|
|
"rewards/rejected": 1.136588454246521,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3392462521650839,
|
|
"grad_norm": 48.5,
|
|
"ht_mnpo/logit": 0.172264963388443,
|
|
"ht_mnpo/residual": 0.16476497054100037,
|
|
"ht_mnpo/residual_abs": 0.2448660433292389,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -2.225900650024414,
|
|
"logits/rejected": -1.9668943881988525,
|
|
"logps/chosen": -0.2590000033378601,
|
|
"logps/rejected": -0.2526404559612274,
|
|
"loss": 0.5311131477355957,
|
|
"loss/core": 0.5311131477355957,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.443131923675537,
|
|
"rewards/margins": 1.7226498126983643,
|
|
"rewards/rejected": 1.7204822301864624,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.3440243683927611,
|
|
"grad_norm": 3.640625,
|
|
"ht_mnpo/logit": 0.1497168242931366,
|
|
"ht_mnpo/residual": 0.14221683144569397,
|
|
"ht_mnpo/residual_abs": 0.18900860846042633,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -2.070448875427246,
|
|
"logits/rejected": -1.9167006015777588,
|
|
"logps/chosen": -0.2813272476196289,
|
|
"logps/rejected": -0.2787591516971588,
|
|
"loss": 0.5066571831703186,
|
|
"loss/core": 0.5066571831703186,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.164358377456665,
|
|
"rewards/margins": 1.4971683025360107,
|
|
"rewards/rejected": 1.6671901941299438,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.3488024846204384,
|
|
"grad_norm": 4.40625,
|
|
"ht_mnpo/logit": 0.135546013712883,
|
|
"ht_mnpo/residual": 0.12804600596427917,
|
|
"ht_mnpo/residual_abs": 0.15786965191364288,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -2.3442587852478027,
|
|
"logits/rejected": -2.12084698677063,
|
|
"logps/chosen": -0.2686806619167328,
|
|
"logps/rejected": -0.27344799041748047,
|
|
"loss": 0.699191689491272,
|
|
"loss/core": 0.699191689491272,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.5767464637756348,
|
|
"rewards/margins": 1.3554600477218628,
|
|
"rewards/rejected": 1.2212862968444824,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.35358060084811566,
|
|
"grad_norm": 1.859375,
|
|
"ht_mnpo/logit": 0.09349587559700012,
|
|
"ht_mnpo/residual": 0.0859958678483963,
|
|
"ht_mnpo/residual_abs": 0.10029371827840805,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.2674036026000977,
|
|
"logits/rejected": -1.944549560546875,
|
|
"logps/chosen": -0.3023375868797302,
|
|
"logps/rejected": -0.3050137162208557,
|
|
"loss": 0.2242615520954132,
|
|
"loss/core": 0.2242615520954132,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 1.805890440940857,
|
|
"rewards/margins": 0.9349587559700012,
|
|
"rewards/rejected": 0.8709318041801453,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35835871707579287,
|
|
"grad_norm": 8.8125,
|
|
"ht_mnpo/logit": 0.09880022704601288,
|
|
"ht_mnpo/residual": 0.09130023419857025,
|
|
"ht_mnpo/residual_abs": 0.12248305231332779,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -1.9330936670303345,
|
|
"logits/rejected": -1.8162342309951782,
|
|
"logps/chosen": -0.3033140003681183,
|
|
"logps/rejected": -0.27995291352272034,
|
|
"loss": 0.2245170623064041,
|
|
"loss/core": 0.2245170623064041,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.7184536457061768,
|
|
"rewards/margins": 0.9880023002624512,
|
|
"rewards/rejected": 1.730450987815857,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36313683330347013,
|
|
"grad_norm": 8.0625,
|
|
"ht_mnpo/logit": 0.13801223039627075,
|
|
"ht_mnpo/residual": 0.13051220774650574,
|
|
"ht_mnpo/residual_abs": 0.13972285389900208,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -2.269474506378174,
|
|
"logits/rejected": -1.9879837036132812,
|
|
"logps/chosen": -0.29586416482925415,
|
|
"logps/rejected": -0.303752064704895,
|
|
"loss": 0.2670932412147522,
|
|
"loss/core": 0.2670932412147522,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.13942289352417,
|
|
"rewards/margins": 1.3801223039627075,
|
|
"rewards/rejected": 0.7593008279800415,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36791494953114734,
|
|
"grad_norm": 8.3125,
|
|
"ht_mnpo/logit": 0.07426253706216812,
|
|
"ht_mnpo/residual": 0.0667625367641449,
|
|
"ht_mnpo/residual_abs": 0.08736227452754974,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -2.2373242378234863,
|
|
"logits/rejected": -2.0958285331726074,
|
|
"logps/chosen": -0.2665890157222748,
|
|
"logps/rejected": -0.2846522033214569,
|
|
"loss": 0.03473305329680443,
|
|
"loss/core": 0.03473305329680443,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7353073358535767,
|
|
"rewards/margins": 0.74262535572052,
|
|
"rewards/rejected": 0.9926820993423462,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726930657588246,
|
|
"grad_norm": 0.98828125,
|
|
"ht_mnpo/logit": 0.11850671470165253,
|
|
"ht_mnpo/residual": 0.1110067218542099,
|
|
"ht_mnpo/residual_abs": 0.1288645714521408,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -2.2801146507263184,
|
|
"logits/rejected": -2.045973300933838,
|
|
"logps/chosen": -0.29588863253593445,
|
|
"logps/rejected": -0.26954060792922974,
|
|
"loss": 0.3808225989341736,
|
|
"loss/core": 0.3808225989341736,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.7486746311187744,
|
|
"rewards/margins": 1.1850672960281372,
|
|
"rewards/rejected": 1.5636073350906372,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.3774711819865018,
|
|
"grad_norm": 25.0,
|
|
"ht_mnpo/logit": 0.2065129578113556,
|
|
"ht_mnpo/residual": 0.19901293516159058,
|
|
"ht_mnpo/residual_abs": 0.24453966319561005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -2.167894124984741,
|
|
"logits/rejected": -2.0230815410614014,
|
|
"logps/chosen": -0.2743033766746521,
|
|
"logps/rejected": -0.3099581003189087,
|
|
"loss": 0.7823916673660278,
|
|
"loss/core": 0.7823916673660278,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2027816772460938,
|
|
"rewards/margins": 2.065129280090332,
|
|
"rewards/rejected": 1.1376521587371826,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.3822492982141791,
|
|
"grad_norm": 1.9765625,
|
|
"ht_mnpo/logit": 0.04907558485865593,
|
|
"ht_mnpo/residual": 0.041575588285923004,
|
|
"ht_mnpo/residual_abs": 0.056028060615062714,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.0534815788269043,
|
|
"logits/rejected": -1.87503182888031,
|
|
"logps/chosen": -0.2717309594154358,
|
|
"logps/rejected": -0.2837705910205841,
|
|
"loss": 0.010935614816844463,
|
|
"loss/core": 0.010935614816844463,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.9626553058624268,
|
|
"rewards/margins": 0.4907558560371399,
|
|
"rewards/rejected": 1.4718996286392212,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.3870274144418563,
|
|
"grad_norm": 12.5625,
|
|
"ht_mnpo/logit": 0.13887158036231995,
|
|
"ht_mnpo/residual": 0.13137157261371613,
|
|
"ht_mnpo/residual_abs": 0.22304904460906982,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -2.0426554679870605,
|
|
"logits/rejected": -1.777740478515625,
|
|
"logps/chosen": -0.2529454827308655,
|
|
"logps/rejected": -0.2843603491783142,
|
|
"loss": 0.6288084983825684,
|
|
"loss/core": 0.6288084983825684,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 3.3556621074676514,
|
|
"rewards/margins": 1.3887159824371338,
|
|
"rewards/rejected": 1.9669462442398071,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39180553066953355,
|
|
"grad_norm": 0.4140625,
|
|
"ht_mnpo/logit": 0.11563078314065933,
|
|
"ht_mnpo/residual": 0.1081307902932167,
|
|
"ht_mnpo/residual_abs": 0.13895943760871887,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.381803035736084,
|
|
"logits/rejected": -2.0271756649017334,
|
|
"logps/chosen": -0.2997882664203644,
|
|
"logps/rejected": -0.3019891381263733,
|
|
"loss": 0.2540455758571625,
|
|
"loss/core": 0.2540455758571625,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 2.6567492485046387,
|
|
"rewards/margins": 1.1563078165054321,
|
|
"rewards/rejected": 1.5004416704177856,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.39658364689721076,
|
|
"grad_norm": 8.3125,
|
|
"ht_mnpo/logit": 0.16058483719825745,
|
|
"ht_mnpo/residual": 0.15308482944965363,
|
|
"ht_mnpo/residual_abs": 0.17841237783432007,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -2.159789800643921,
|
|
"logits/rejected": -1.9371411800384521,
|
|
"logps/chosen": -0.2774130403995514,
|
|
"logps/rejected": -0.2748417258262634,
|
|
"loss": 0.4761256277561188,
|
|
"loss/core": 0.4761256277561188,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.7579145431518555,
|
|
"rewards/margins": 1.6058483123779297,
|
|
"rewards/rejected": 1.1520664691925049,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.401361763124888,
|
|
"grad_norm": 360.0,
|
|
"ht_mnpo/logit": 0.17204149067401886,
|
|
"ht_mnpo/residual": 0.16454148292541504,
|
|
"ht_mnpo/residual_abs": 0.25641435384750366,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -1.9525874853134155,
|
|
"logits/rejected": -1.7593332529067993,
|
|
"logps/chosen": -0.2873152196407318,
|
|
"logps/rejected": -0.28734180331230164,
|
|
"loss": 0.5311289429664612,
|
|
"loss/core": 0.5311289429664612,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.404254913330078,
|
|
"rewards/margins": 1.7204147577285767,
|
|
"rewards/rejected": 1.6838403940200806,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.40613987935256524,
|
|
"grad_norm": 1112.0,
|
|
"ht_mnpo/logit": 0.13960780203342438,
|
|
"ht_mnpo/residual": 0.13210779428482056,
|
|
"ht_mnpo/residual_abs": 0.2718185782432556,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -2.2387444972991943,
|
|
"logits/rejected": -2.000307083129883,
|
|
"logps/chosen": -0.2742324471473694,
|
|
"logps/rejected": -0.28232383728027344,
|
|
"loss": 0.9235121607780457,
|
|
"loss/core": 0.9235121607780457,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 3.1787946224212646,
|
|
"rewards/margins": 1.3960778713226318,
|
|
"rewards/rejected": 1.7827165126800537,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.4109179955802425,
|
|
"grad_norm": 306.0,
|
|
"ht_mnpo/logit": 0.03273645043373108,
|
|
"ht_mnpo/residual": 0.025236451998353004,
|
|
"ht_mnpo/residual_abs": 0.08493396639823914,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.2073371410369873,
|
|
"logits/rejected": -1.9156125783920288,
|
|
"logps/chosen": -0.284179151058197,
|
|
"logps/rejected": -0.2822193503379822,
|
|
"loss": 0.08687691390514374,
|
|
"loss/core": 0.08687691390514374,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.3117029666900635,
|
|
"rewards/margins": 0.3273644745349884,
|
|
"rewards/rejected": 0.9843384027481079,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.4156961118079197,
|
|
"grad_norm": 1040.0,
|
|
"ht_mnpo/logit": 0.147271066904068,
|
|
"ht_mnpo/residual": 0.13977108895778656,
|
|
"ht_mnpo/residual_abs": 0.2996138334274292,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.086392641067505,
|
|
"logits/rejected": -1.9442352056503296,
|
|
"logps/chosen": -0.30270153284072876,
|
|
"logps/rejected": -0.33257466554641724,
|
|
"loss": 1.1545547246932983,
|
|
"loss/core": 1.1545547246932983,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.7823562622070312,
|
|
"rewards/margins": 1.4727110862731934,
|
|
"rewards/rejected": 2.309644937515259,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.420474228035597,
|
|
"grad_norm": 5.09375,
|
|
"ht_mnpo/logit": 0.15315040946006775,
|
|
"ht_mnpo/residual": 0.14565041661262512,
|
|
"ht_mnpo/residual_abs": 0.2222578078508377,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -2.4342799186706543,
|
|
"logits/rejected": -2.189716100692749,
|
|
"logps/chosen": -0.27770087122917175,
|
|
"logps/rejected": -0.291808158159256,
|
|
"loss": 0.6261534690856934,
|
|
"loss/core": 0.6261534690856934,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.841123104095459,
|
|
"rewards/margins": 1.5315040349960327,
|
|
"rewards/rejected": 1.3096191883087158,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.4252523442632742,
|
|
"grad_norm": 5.125,
|
|
"ht_mnpo/logit": 0.0066137416288256645,
|
|
"ht_mnpo/residual": -0.0008862599497660995,
|
|
"ht_mnpo/residual_abs": 0.1915988177061081,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -2.1221556663513184,
|
|
"logits/rejected": -1.8717864751815796,
|
|
"logps/chosen": -0.30727943778038025,
|
|
"logps/rejected": -0.3041136562824249,
|
|
"loss": 0.40154901146888733,
|
|
"loss/core": 0.40154901146888733,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.6310601234436035,
|
|
"rewards/margins": 0.06613747030496597,
|
|
"rewards/rejected": 2.56492280960083,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.43003046049095145,
|
|
"grad_norm": 4.5625,
|
|
"ht_mnpo/logit": 0.028625909239053726,
|
|
"ht_mnpo/residual": 0.021125907078385353,
|
|
"ht_mnpo/residual_abs": 0.06126154586672783,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -2.1283726692199707,
|
|
"logits/rejected": -1.7972466945648193,
|
|
"logps/chosen": -0.2941470742225647,
|
|
"logps/rejected": -0.31552690267562866,
|
|
"loss": 0.015647191554307938,
|
|
"loss/core": 0.015647191554307938,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 1.5162620544433594,
|
|
"rewards/margins": 0.28625908493995667,
|
|
"rewards/rejected": 1.2300031185150146,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.43480857671862866,
|
|
"grad_norm": 7.875,
|
|
"ht_mnpo/logit": -0.03804566711187363,
|
|
"ht_mnpo/residual": -0.04554566740989685,
|
|
"ht_mnpo/residual_abs": 0.13553011417388916,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -2.3924567699432373,
|
|
"logits/rejected": -2.0088376998901367,
|
|
"logps/chosen": -0.3002735674381256,
|
|
"logps/rejected": -0.3013390600681305,
|
|
"loss": 0.30527040362358093,
|
|
"loss/core": 0.30527040362358093,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.0105655193328857,
|
|
"rewards/margins": -0.3804566264152527,
|
|
"rewards/rejected": 1.3910223245620728,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395866929463059,
|
|
"grad_norm": 3.1875,
|
|
"ht_mnpo/logit": 0.020327895879745483,
|
|
"ht_mnpo/residual": 0.012827901169657707,
|
|
"ht_mnpo/residual_abs": 0.11514680087566376,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -2.346726894378662,
|
|
"logits/rejected": -1.9906597137451172,
|
|
"logps/chosen": -0.2989094853401184,
|
|
"logps/rejected": -0.3026769757270813,
|
|
"loss": 0.1966794729232788,
|
|
"loss/core": 0.1966794729232788,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.6205774545669556,
|
|
"rewards/margins": 0.20327889919281006,
|
|
"rewards/rejected": 1.417298674583435,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44436480917398313,
|
|
"grad_norm": 9.9375,
|
|
"ht_mnpo/logit": 0.01999581605195999,
|
|
"ht_mnpo/residual": 0.012495816685259342,
|
|
"ht_mnpo/residual_abs": 0.11453278362751007,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.9809582233428955,
|
|
"logits/rejected": -1.6697006225585938,
|
|
"logps/chosen": -0.2852214276790619,
|
|
"logps/rejected": -0.29375481605529785,
|
|
"loss": 0.09421398490667343,
|
|
"loss/core": 0.09421398490667343,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.281210422515869,
|
|
"rewards/margins": 0.19995816051959991,
|
|
"rewards/rejected": 2.081252336502075,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4491429254016604,
|
|
"grad_norm": 804.0,
|
|
"ht_mnpo/logit": 0.07451383024454117,
|
|
"ht_mnpo/residual": 0.06701384484767914,
|
|
"ht_mnpo/residual_abs": 0.19596199691295624,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -2.164752721786499,
|
|
"logits/rejected": -1.8490082025527954,
|
|
"logps/chosen": -0.25757962465286255,
|
|
"logps/rejected": -0.2674221098423004,
|
|
"loss": 0.694079577922821,
|
|
"loss/core": 0.694079577922821,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4484660625457764,
|
|
"rewards/margins": 0.74513840675354,
|
|
"rewards/rejected": 1.7033277750015259,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.4539210416293376,
|
|
"grad_norm": 21.5,
|
|
"ht_mnpo/logit": 0.05053982138633728,
|
|
"ht_mnpo/residual": 0.043039821088314056,
|
|
"ht_mnpo/residual_abs": 0.17135195434093475,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.3393898010253906,
|
|
"logits/rejected": -1.9308264255523682,
|
|
"logps/chosen": -0.27896398305892944,
|
|
"logps/rejected": -0.2943103611469269,
|
|
"loss": 0.2023368626832962,
|
|
"loss/core": 0.2023368626832962,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.079939365386963,
|
|
"rewards/margins": 0.5053982734680176,
|
|
"rewards/rejected": 1.5745410919189453,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45869915785701487,
|
|
"grad_norm": 145.0,
|
|
"ht_mnpo/logit": 0.07996602356433868,
|
|
"ht_mnpo/residual": 0.07246602326631546,
|
|
"ht_mnpo/residual_abs": 0.15748628973960876,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -2.064171314239502,
|
|
"logits/rejected": -1.7457411289215088,
|
|
"logps/chosen": -0.28379932045936584,
|
|
"logps/rejected": -0.30818262696266174,
|
|
"loss": 0.26114827394485474,
|
|
"loss/core": 0.26114827394485474,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.5865588188171387,
|
|
"rewards/margins": 0.799660325050354,
|
|
"rewards/rejected": 1.7868983745574951,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.46347727408469214,
|
|
"grad_norm": 2.625,
|
|
"ht_mnpo/logit": 0.062350332736968994,
|
|
"ht_mnpo/residual": 0.05485032871365547,
|
|
"ht_mnpo/residual_abs": 0.08066234737634659,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.3802075386047363,
|
|
"logits/rejected": -2.1280293464660645,
|
|
"logps/chosen": -0.2738611102104187,
|
|
"logps/rejected": -0.289838969707489,
|
|
"loss": 0.0550173744559288,
|
|
"loss/core": 0.0550173744559288,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4340312480926514,
|
|
"rewards/margins": 0.6235032677650452,
|
|
"rewards/rejected": 0.8105279803276062,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.46825539031236935,
|
|
"grad_norm": 41.0,
|
|
"ht_mnpo/logit": 0.07819999009370804,
|
|
"ht_mnpo/residual": 0.07069997489452362,
|
|
"ht_mnpo/residual_abs": 0.1625690758228302,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.249577522277832,
|
|
"logits/rejected": -2.0636165142059326,
|
|
"logps/chosen": -0.2568047046661377,
|
|
"logps/rejected": -0.3003430962562561,
|
|
"loss": 0.22979728877544403,
|
|
"loss/core": 0.22979728877544403,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.4754576683044434,
|
|
"rewards/margins": 0.7819998264312744,
|
|
"rewards/rejected": 1.693457841873169,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.4730335065400466,
|
|
"grad_norm": 6.25,
|
|
"ht_mnpo/logit": 0.06184772402048111,
|
|
"ht_mnpo/residual": 0.05434773117303848,
|
|
"ht_mnpo/residual_abs": 0.08136261999607086,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.36091947555542,
|
|
"logits/rejected": -2.0397167205810547,
|
|
"logps/chosen": -0.2872092127799988,
|
|
"logps/rejected": -0.284702330827713,
|
|
"loss": 0.059263549745082855,
|
|
"loss/core": 0.059263549745082855,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.4394937753677368,
|
|
"rewards/margins": 0.6184772253036499,
|
|
"rewards/rejected": 0.8210164904594421,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.4778116227677238,
|
|
"grad_norm": 1.7734375,
|
|
"ht_mnpo/logit": 0.05665602162480354,
|
|
"ht_mnpo/residual": 0.04915601760149002,
|
|
"ht_mnpo/residual_abs": 0.1744033843278885,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -2.30210280418396,
|
|
"logits/rejected": -2.0592434406280518,
|
|
"logps/chosen": -0.28479260206222534,
|
|
"logps/rejected": -0.295644611120224,
|
|
"loss": 0.4020271897315979,
|
|
"loss/core": 0.4020271897315979,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8941959142684937,
|
|
"rewards/margins": 0.5665602684020996,
|
|
"rewards/rejected": 1.3276355266571045,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825897389954011,
|
|
"grad_norm": 14.1875,
|
|
"ht_mnpo/logit": 0.043277788907289505,
|
|
"ht_mnpo/residual": 0.03577778860926628,
|
|
"ht_mnpo/residual_abs": 0.0675283819437027,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -2.371837854385376,
|
|
"logits/rejected": -2.1066370010375977,
|
|
"logps/chosen": -0.260089635848999,
|
|
"logps/rejected": -0.28670915961265564,
|
|
"loss": 0.03348282352089882,
|
|
"loss/core": 0.03348282352089882,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.3851932287216187,
|
|
"rewards/margins": 0.43277788162231445,
|
|
"rewards/rejected": 0.9524153470993042,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4873678552230783,
|
|
"grad_norm": 572.0,
|
|
"ht_mnpo/logit": 0.0772649273276329,
|
|
"ht_mnpo/residual": 0.06976493448019028,
|
|
"ht_mnpo/residual_abs": 0.23304533958435059,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -2.180732011795044,
|
|
"logits/rejected": -1.9350513219833374,
|
|
"logps/chosen": -0.2900449335575104,
|
|
"logps/rejected": -0.3275887370109558,
|
|
"loss": 0.701738178730011,
|
|
"loss/core": 0.701738178730011,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.6179404258728027,
|
|
"rewards/margins": 0.7726494073867798,
|
|
"rewards/rejected": 1.8452911376953125,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49214597145075556,
|
|
"grad_norm": 2080.0,
|
|
"ht_mnpo/logit": 0.13127067685127258,
|
|
"ht_mnpo/residual": 0.12377066910266876,
|
|
"ht_mnpo/residual_abs": 0.1691356599330902,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -2.2999091148376465,
|
|
"logits/rejected": -1.9634205102920532,
|
|
"logps/chosen": -0.29252487421035767,
|
|
"logps/rejected": -0.29366955161094666,
|
|
"loss": 0.72135990858078,
|
|
"loss/core": 0.72135990858078,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4953925609588623,
|
|
"rewards/margins": 1.312706708908081,
|
|
"rewards/rejected": 1.1826857328414917,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.49692408767843277,
|
|
"grad_norm": 2.390625,
|
|
"ht_mnpo/logit": 0.028129365295171738,
|
|
"ht_mnpo/residual": 0.02062937058508396,
|
|
"ht_mnpo/residual_abs": 0.12287096679210663,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -2.214996576309204,
|
|
"logits/rejected": -1.9160572290420532,
|
|
"logps/chosen": -0.2983536124229431,
|
|
"logps/rejected": -0.31177738308906555,
|
|
"loss": 0.20848357677459717,
|
|
"loss/core": 0.20848357677459717,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 1.8202794790267944,
|
|
"rewards/margins": 0.28129369020462036,
|
|
"rewards/rejected": 1.5389859676361084,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.50170220390611,
|
|
"grad_norm": 800.0,
|
|
"ht_mnpo/logit": 0.19240263104438782,
|
|
"ht_mnpo/residual": 0.184902623295784,
|
|
"ht_mnpo/residual_abs": 0.3580849766731262,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.0721850395202637,
|
|
"logits/rejected": -1.8336656093597412,
|
|
"logps/chosen": -0.27473676204681396,
|
|
"logps/rejected": -0.2788922190666199,
|
|
"loss": 1.030470848083496,
|
|
"loss/core": 1.030470848083496,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.851513862609863,
|
|
"rewards/margins": 1.9240261316299438,
|
|
"rewards/rejected": 2.927487850189209,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5064803201337873,
|
|
"grad_norm": 13.3125,
|
|
"ht_mnpo/logit": 0.06076349690556526,
|
|
"ht_mnpo/residual": 0.05326349288225174,
|
|
"ht_mnpo/residual_abs": 0.08253750950098038,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.1719086170196533,
|
|
"logits/rejected": -1.937659502029419,
|
|
"logps/chosen": -0.29302453994750977,
|
|
"logps/rejected": -0.3102428913116455,
|
|
"loss": 0.044716306030750275,
|
|
"loss/core": 0.044716306030750275,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.4354779720306396,
|
|
"rewards/margins": 0.6076349020004272,
|
|
"rewards/rejected": 1.8278430700302124,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5112584363614645,
|
|
"grad_norm": 167.0,
|
|
"ht_mnpo/logit": 0.06611252576112747,
|
|
"ht_mnpo/residual": 0.05861252546310425,
|
|
"ht_mnpo/residual_abs": 0.11790484189987183,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -2.1390864849090576,
|
|
"logits/rejected": -1.9728997945785522,
|
|
"logps/chosen": -0.28939175605773926,
|
|
"logps/rejected": -0.29434826970100403,
|
|
"loss": 0.09479711949825287,
|
|
"loss/core": 0.09479711949825287,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.8622554540634155,
|
|
"rewards/margins": 0.6611253023147583,
|
|
"rewards/rejected": 1.2011301517486572,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5160365525891417,
|
|
"grad_norm": 9.875,
|
|
"ht_mnpo/logit": 0.10702864080667496,
|
|
"ht_mnpo/residual": 0.09952862560749054,
|
|
"ht_mnpo/residual_abs": 0.1450154036283493,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.327497959136963,
|
|
"logits/rejected": -2.0987296104431152,
|
|
"logps/chosen": -0.29696059226989746,
|
|
"logps/rejected": -0.3068595230579376,
|
|
"loss": 0.19229401648044586,
|
|
"loss/core": 0.19229401648044586,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 2.24061918258667,
|
|
"rewards/margins": 1.0702862739562988,
|
|
"rewards/rejected": 1.1703331470489502,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.5208146688168189,
|
|
"grad_norm": 12.5,
|
|
"ht_mnpo/logit": 0.04363741725683212,
|
|
"ht_mnpo/residual": 0.0361374206840992,
|
|
"ht_mnpo/residual_abs": 0.07851780205965042,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.2913150787353516,
|
|
"logits/rejected": -2.051034450531006,
|
|
"logps/chosen": -0.3131157159805298,
|
|
"logps/rejected": -0.3089737296104431,
|
|
"loss": 0.04530810937285423,
|
|
"loss/core": 0.04530810937285423,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.512770414352417,
|
|
"rewards/margins": 0.4363742470741272,
|
|
"rewards/rejected": 1.0763962268829346,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5255927850444962,
|
|
"grad_norm": 9.0,
|
|
"ht_mnpo/logit": 0.06660556048154831,
|
|
"ht_mnpo/residual": 0.059105563908815384,
|
|
"ht_mnpo/residual_abs": 0.07340370863676071,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -2.189758062362671,
|
|
"logits/rejected": -2.015587568283081,
|
|
"logps/chosen": -0.29240942001342773,
|
|
"logps/rejected": -0.31375378370285034,
|
|
"loss": 0.03384558483958244,
|
|
"loss/core": 0.03384558483958244,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.594940185546875,
|
|
"rewards/margins": 0.6660556197166443,
|
|
"rewards/rejected": 0.9288846850395203,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5303709012721735,
|
|
"grad_norm": 3.078125,
|
|
"ht_mnpo/logit": 0.06365452706813812,
|
|
"ht_mnpo/residual": 0.0561545304954052,
|
|
"ht_mnpo/residual_abs": 0.08707477897405624,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.128129005432129,
|
|
"logits/rejected": -2.004274368286133,
|
|
"logps/chosen": -0.3144676983356476,
|
|
"logps/rejected": -0.32734498381614685,
|
|
"loss": 0.04949260503053665,
|
|
"loss/core": 0.04949260503053665,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6513652801513672,
|
|
"rewards/margins": 0.6365452408790588,
|
|
"rewards/rejected": 1.014819860458374,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5351490174998507,
|
|
"grad_norm": 0.72265625,
|
|
"ht_mnpo/logit": -0.0056523485109210014,
|
|
"ht_mnpo/residual": -0.013152359053492546,
|
|
"ht_mnpo/residual_abs": 0.2248121052980423,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -2.174121141433716,
|
|
"logits/rejected": -1.9470113515853882,
|
|
"logps/chosen": -0.28464481234550476,
|
|
"logps/rejected": -0.3088465631008148,
|
|
"loss": 0.7891527414321899,
|
|
"loss/core": 0.7891527414321899,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.3156628608703613,
|
|
"rewards/margins": -0.05652338266372681,
|
|
"rewards/rejected": 2.3721861839294434,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5399271337275279,
|
|
"grad_norm": 7.9375,
|
|
"ht_mnpo/logit": 0.002409947570413351,
|
|
"ht_mnpo/residual": -0.005090050399303436,
|
|
"ht_mnpo/residual_abs": 0.06951995939016342,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.351963520050049,
|
|
"logits/rejected": -2.055279016494751,
|
|
"logps/chosen": -0.3088880181312561,
|
|
"logps/rejected": -0.32034802436828613,
|
|
"loss": 0.04919041693210602,
|
|
"loss/core": 0.04919041693210602,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 1.2688082456588745,
|
|
"rewards/margins": 0.024099458009004593,
|
|
"rewards/rejected": 1.244708776473999,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5447052499552052,
|
|
"grad_norm": 3.375,
|
|
"ht_mnpo/logit": 0.051477622240781784,
|
|
"ht_mnpo/residual": 0.04397762194275856,
|
|
"ht_mnpo/residual_abs": 0.13583263754844666,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -2.2267661094665527,
|
|
"logits/rejected": -2.0639772415161133,
|
|
"logps/chosen": -0.29423585534095764,
|
|
"logps/rejected": -0.31717735528945923,
|
|
"loss": 0.18170244991779327,
|
|
"loss/core": 0.18170244991779327,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9906375408172607,
|
|
"rewards/margins": 0.5147761106491089,
|
|
"rewards/rejected": 1.4758613109588623,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5494833661828824,
|
|
"grad_norm": 14.75,
|
|
"ht_mnpo/logit": 0.05904921144247055,
|
|
"ht_mnpo/residual": 0.05154920369386673,
|
|
"ht_mnpo/residual_abs": 0.09378327429294586,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.3841331005096436,
|
|
"logits/rejected": -2.1283442974090576,
|
|
"logps/chosen": -0.2951911687850952,
|
|
"logps/rejected": -0.29570120573043823,
|
|
"loss": 0.07935883849859238,
|
|
"loss/core": 0.07935883849859238,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.4893664121627808,
|
|
"rewards/margins": 0.5904920101165771,
|
|
"rewards/rejected": 0.8988744616508484,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5542614824105596,
|
|
"grad_norm": 20.875,
|
|
"ht_mnpo/logit": 0.08906254172325134,
|
|
"ht_mnpo/residual": 0.08156253397464752,
|
|
"ht_mnpo/residual_abs": 0.1606951355934143,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.1410086154937744,
|
|
"logits/rejected": -1.9277909994125366,
|
|
"logps/chosen": -0.296114057302475,
|
|
"logps/rejected": -0.3074384927749634,
|
|
"loss": 0.1655869483947754,
|
|
"loss/core": 0.1655869483947754,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.5124917030334473,
|
|
"rewards/margins": 0.8906253576278687,
|
|
"rewards/rejected": 1.6218664646148682,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5590395986382368,
|
|
"grad_norm": 21.375,
|
|
"ht_mnpo/logit": 0.11307352781295776,
|
|
"ht_mnpo/residual": 0.10557352006435394,
|
|
"ht_mnpo/residual_abs": 0.11429446935653687,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -2.2346272468566895,
|
|
"logits/rejected": -1.9701683521270752,
|
|
"logps/chosen": -0.2883383631706238,
|
|
"logps/rejected": -0.3066983222961426,
|
|
"loss": 0.09720277786254883,
|
|
"loss/core": 0.09720277786254883,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.7855782508850098,
|
|
"rewards/margins": 1.130735158920288,
|
|
"rewards/rejected": 1.6548430919647217,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5638177148659141,
|
|
"grad_norm": 165.0,
|
|
"ht_mnpo/logit": 0.07173161953687668,
|
|
"ht_mnpo/residual": 0.06423161923885345,
|
|
"ht_mnpo/residual_abs": 0.13386103510856628,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -2.5045523643493652,
|
|
"logits/rejected": -2.1490824222564697,
|
|
"logps/chosen": -0.2741159200668335,
|
|
"logps/rejected": -0.2885042428970337,
|
|
"loss": 0.2545531392097473,
|
|
"loss/core": 0.2545531392097473,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.7796194553375244,
|
|
"rewards/margins": 0.7173162698745728,
|
|
"rewards/rejected": 1.0623031854629517,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5685958310935914,
|
|
"grad_norm": 11.9375,
|
|
"ht_mnpo/logit": 0.051647841930389404,
|
|
"ht_mnpo/residual": 0.04414784163236618,
|
|
"ht_mnpo/residual_abs": 0.05486813187599182,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -2.4213550090789795,
|
|
"logits/rejected": -2.244135618209839,
|
|
"logps/chosen": -0.2851046323776245,
|
|
"logps/rejected": -0.28078338503837585,
|
|
"loss": 0.023371253162622452,
|
|
"loss/core": 0.023371253162622452,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.1057530641555786,
|
|
"rewards/margins": 0.516478419303894,
|
|
"rewards/rejected": 0.5892745852470398,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5733739473212686,
|
|
"grad_norm": 42.0,
|
|
"ht_mnpo/logit": 0.042090535163879395,
|
|
"ht_mnpo/residual": 0.03459053486585617,
|
|
"ht_mnpo/residual_abs": 0.13397017121315002,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -2.1188714504241943,
|
|
"logits/rejected": -2.0343403816223145,
|
|
"logps/chosen": -0.27570241689682007,
|
|
"logps/rejected": -0.2801577150821686,
|
|
"loss": 0.20747938752174377,
|
|
"loss/core": 0.20747938752174377,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7335128784179688,
|
|
"rewards/margins": 0.42090529203414917,
|
|
"rewards/rejected": 1.3126074075698853,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5781520635489459,
|
|
"grad_norm": 50.0,
|
|
"ht_mnpo/logit": 0.13987644016742706,
|
|
"ht_mnpo/residual": 0.13237643241882324,
|
|
"ht_mnpo/residual_abs": 0.1692580282688141,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -2.182706832885742,
|
|
"logits/rejected": -2.002788543701172,
|
|
"logps/chosen": -0.2711128294467926,
|
|
"logps/rejected": -0.28066202998161316,
|
|
"loss": 0.31835612654685974,
|
|
"loss/core": 0.31835612654685974,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.817673444747925,
|
|
"rewards/margins": 1.3987643718719482,
|
|
"rewards/rejected": 1.4189090728759766,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5829301797766231,
|
|
"grad_norm": 2.625,
|
|
"ht_mnpo/logit": 0.05944681912660599,
|
|
"ht_mnpo/residual": 0.051946818828582764,
|
|
"ht_mnpo/residual_abs": 0.11564245074987411,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -2.3501338958740234,
|
|
"logits/rejected": -2.1967575550079346,
|
|
"logps/chosen": -0.25377607345581055,
|
|
"logps/rejected": -0.26492077112197876,
|
|
"loss": 0.06198473647236824,
|
|
"loss/core": 0.06198473647236824,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9941737651824951,
|
|
"rewards/margins": 0.5944682359695435,
|
|
"rewards/rejected": 1.3997056484222412,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.5877082960043003,
|
|
"grad_norm": 101.5,
|
|
"ht_mnpo/logit": 0.023357708007097244,
|
|
"ht_mnpo/residual": 0.015857713297009468,
|
|
"ht_mnpo/residual_abs": 0.14009633660316467,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -2.2582335472106934,
|
|
"logits/rejected": -2.0130774974823,
|
|
"logps/chosen": -0.26372048258781433,
|
|
"logps/rejected": -0.26262927055358887,
|
|
"loss": 0.11254549026489258,
|
|
"loss/core": 0.11254549026489258,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 2.158015012741089,
|
|
"rewards/margins": 0.23357710242271423,
|
|
"rewards/rejected": 1.9244378805160522,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5924864122319775,
|
|
"grad_norm": 4.09375,
|
|
"ht_mnpo/logit": 0.03670644760131836,
|
|
"ht_mnpo/residual": 0.029206443578004837,
|
|
"ht_mnpo/residual_abs": 0.10636608302593231,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.3275980949401855,
|
|
"logits/rejected": -2.0934624671936035,
|
|
"logps/chosen": -0.2519906163215637,
|
|
"logps/rejected": -0.26454100012779236,
|
|
"loss": 0.08830438554286957,
|
|
"loss/core": 0.08830438554286957,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.702371597290039,
|
|
"rewards/margins": 0.3670644164085388,
|
|
"rewards/rejected": 1.3353068828582764,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5972645284596548,
|
|
"grad_norm": 5.9375,
|
|
"ht_mnpo/logit": 0.10891832411289215,
|
|
"ht_mnpo/residual": 0.10141833126544952,
|
|
"ht_mnpo/residual_abs": 0.18419775366783142,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -2.145535469055176,
|
|
"logits/rejected": -1.952048659324646,
|
|
"logps/chosen": -0.2804136872291565,
|
|
"logps/rejected": -0.2959136664867401,
|
|
"loss": 0.693868100643158,
|
|
"loss/core": 0.693868100643158,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.70146107673645,
|
|
"rewards/margins": 1.0891832113265991,
|
|
"rewards/rejected": 1.6122777462005615,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.602042644687332,
|
|
"grad_norm": 6.875,
|
|
"ht_mnpo/logit": 0.1092601791024208,
|
|
"ht_mnpo/residual": 0.10176018625497818,
|
|
"ht_mnpo/residual_abs": 0.25469109416007996,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -2.1239399909973145,
|
|
"logits/rejected": -1.8291689157485962,
|
|
"logps/chosen": -0.2866722047328949,
|
|
"logps/rejected": -0.30721598863601685,
|
|
"loss": 0.8774070739746094,
|
|
"loss/core": 0.8774070739746094,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 3.1832964420318604,
|
|
"rewards/margins": 1.0926018953323364,
|
|
"rewards/rejected": 2.0906946659088135,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6068207609150092,
|
|
"grad_norm": 4.1875,
|
|
"ht_mnpo/logit": 0.1126597672700882,
|
|
"ht_mnpo/residual": 0.10515977442264557,
|
|
"ht_mnpo/residual_abs": 0.1412106454372406,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -2.3301758766174316,
|
|
"logits/rejected": -2.0502209663391113,
|
|
"logps/chosen": -0.2599521577358246,
|
|
"logps/rejected": -0.27332812547683716,
|
|
"loss": 0.23787467181682587,
|
|
"loss/core": 0.23787467181682587,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.085704803466797,
|
|
"rewards/margins": 1.1265976428985596,
|
|
"rewards/rejected": 1.9591071605682373,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6115988771426865,
|
|
"grad_norm": 76.0,
|
|
"ht_mnpo/logit": 0.12395603954792023,
|
|
"ht_mnpo/residual": 0.1164560317993164,
|
|
"ht_mnpo/residual_abs": 0.13639070093631744,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.1668243408203125,
|
|
"logits/rejected": -2.0291244983673096,
|
|
"logps/chosen": -0.29634636640548706,
|
|
"logps/rejected": -0.3084748387336731,
|
|
"loss": 0.22144794464111328,
|
|
"loss/core": 0.22144794464111328,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.488347053527832,
|
|
"rewards/margins": 1.2395600080490112,
|
|
"rewards/rejected": 1.2487866878509521,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6163769933703638,
|
|
"grad_norm": 276.0,
|
|
"ht_mnpo/logit": 0.01642979122698307,
|
|
"ht_mnpo/residual": 0.008929798379540443,
|
|
"ht_mnpo/residual_abs": 0.1276627480983734,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -2.2169954776763916,
|
|
"logits/rejected": -1.9446134567260742,
|
|
"logps/chosen": -0.2655065059661865,
|
|
"logps/rejected": -0.2783452868461609,
|
|
"loss": 0.21419398486614227,
|
|
"loss/core": 0.21419398486614227,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.4721295833587646,
|
|
"rewards/margins": 0.16429801285266876,
|
|
"rewards/rejected": 1.3078315258026123,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.621155109598041,
|
|
"grad_norm": 23.5,
|
|
"ht_mnpo/logit": 0.1376076340675354,
|
|
"ht_mnpo/residual": 0.13010765612125397,
|
|
"ht_mnpo/residual_abs": 0.15285083651542664,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -2.1072707176208496,
|
|
"logits/rejected": -1.9887821674346924,
|
|
"logps/chosen": -0.2807113230228424,
|
|
"logps/rejected": -0.2912576496601105,
|
|
"loss": 0.3052452504634857,
|
|
"loss/core": 0.3052452504634857,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.705549955368042,
|
|
"rewards/margins": 1.3760764598846436,
|
|
"rewards/rejected": 1.3294737339019775,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6259332258257182,
|
|
"grad_norm": 66.0,
|
|
"ht_mnpo/logit": 0.03930670768022537,
|
|
"ht_mnpo/residual": 0.031806714832782745,
|
|
"ht_mnpo/residual_abs": 0.07188865542411804,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.338975429534912,
|
|
"logits/rejected": -2.051806688308716,
|
|
"logps/chosen": -0.2882004678249359,
|
|
"logps/rejected": -0.31319910287857056,
|
|
"loss": 0.0360516794025898,
|
|
"loss/core": 0.0360516794025898,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.4952980279922485,
|
|
"rewards/margins": 0.3930671513080597,
|
|
"rewards/rejected": 1.1022307872772217,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6307113420533954,
|
|
"grad_norm": 6.78125,
|
|
"ht_mnpo/logit": 0.056034285575151443,
|
|
"ht_mnpo/residual": 0.04853427782654762,
|
|
"ht_mnpo/residual_abs": 0.07523567974567413,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -2.217348098754883,
|
|
"logits/rejected": -2.0050175189971924,
|
|
"logps/chosen": -0.2928197979927063,
|
|
"logps/rejected": -0.33136898279190063,
|
|
"loss": 0.04290390759706497,
|
|
"loss/core": 0.04290390759706497,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5474987030029297,
|
|
"rewards/margins": 0.5603427886962891,
|
|
"rewards/rejected": 0.9871557950973511,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6354894582810727,
|
|
"grad_norm": 142.0,
|
|
"ht_mnpo/logit": 0.14371220767498016,
|
|
"ht_mnpo/residual": 0.13621219992637634,
|
|
"ht_mnpo/residual_abs": 0.15110185742378235,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -2.243295669555664,
|
|
"logits/rejected": -1.9963970184326172,
|
|
"logps/chosen": -0.2896881401538849,
|
|
"logps/rejected": -0.3032901883125305,
|
|
"loss": 0.4213927388191223,
|
|
"loss/core": 0.4213927388191223,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0614871978759766,
|
|
"rewards/margins": 1.437122106552124,
|
|
"rewards/rejected": 0.6243652701377869,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.6402675745087499,
|
|
"grad_norm": 110.0,
|
|
"ht_mnpo/logit": 0.10394921153783798,
|
|
"ht_mnpo/residual": 0.09644920378923416,
|
|
"ht_mnpo/residual_abs": 0.1331409364938736,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -2.4142959117889404,
|
|
"logits/rejected": -2.069192409515381,
|
|
"logps/chosen": -0.29473966360092163,
|
|
"logps/rejected": -0.31998682022094727,
|
|
"loss": 0.4319857954978943,
|
|
"loss/core": 0.4319857954978943,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7670764923095703,
|
|
"rewards/margins": 1.039492130279541,
|
|
"rewards/rejected": 0.7275844216346741,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6450456907364271,
|
|
"grad_norm": 174.0,
|
|
"ht_mnpo/logit": 0.06010546162724495,
|
|
"ht_mnpo/residual": 0.05260546877980232,
|
|
"ht_mnpo/residual_abs": 0.1043517217040062,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.420896291732788,
|
|
"logits/rejected": -2.177229166030884,
|
|
"logps/chosen": -0.26569315791130066,
|
|
"logps/rejected": -0.2760010063648224,
|
|
"loss": 0.17001619935035706,
|
|
"loss/core": 0.17001619935035706,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.536733865737915,
|
|
"rewards/margins": 0.6010546684265137,
|
|
"rewards/rejected": 0.9356793165206909,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6498238069641044,
|
|
"grad_norm": 5.78125,
|
|
"ht_mnpo/logit": 0.006507441401481628,
|
|
"ht_mnpo/residual": -0.0009925604099407792,
|
|
"ht_mnpo/residual_abs": 0.12537150084972382,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -2.0410115718841553,
|
|
"logits/rejected": -1.7555296421051025,
|
|
"logps/chosen": -0.29283419251441956,
|
|
"logps/rejected": -0.302853524684906,
|
|
"loss": 0.1671266406774521,
|
|
"loss/core": 0.1671266406774521,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.7759294509887695,
|
|
"rewards/margins": 0.06507435441017151,
|
|
"rewards/rejected": 1.7108551263809204,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.6546019231917817,
|
|
"grad_norm": 14.1875,
|
|
"ht_mnpo/logit": -0.01924746111035347,
|
|
"ht_mnpo/residual": -0.026747453957796097,
|
|
"ht_mnpo/residual_abs": 0.21825659275054932,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -2.1334753036499023,
|
|
"logits/rejected": -1.8235372304916382,
|
|
"logps/chosen": -0.2628119885921478,
|
|
"logps/rejected": -0.31102806329727173,
|
|
"loss": 0.42427802085876465,
|
|
"loss/core": 0.42427802085876465,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.3661179542541504,
|
|
"rewards/margins": -0.19247445464134216,
|
|
"rewards/rejected": 2.5585925579071045,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6593800394194589,
|
|
"grad_norm": 21.625,
|
|
"ht_mnpo/logit": 0.061263781040906906,
|
|
"ht_mnpo/residual": 0.053763777017593384,
|
|
"ht_mnpo/residual_abs": 0.07502800226211548,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -2.293973684310913,
|
|
"logits/rejected": -2.080751419067383,
|
|
"logps/chosen": -0.2748379111289978,
|
|
"logps/rejected": -0.2834241986274719,
|
|
"loss": 0.03468334674835205,
|
|
"loss/core": 0.03468334674835205,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.741302251815796,
|
|
"rewards/margins": 0.6126378178596497,
|
|
"rewards/rejected": 1.128664493560791,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6641581556471361,
|
|
"grad_norm": 3.921875,
|
|
"ht_mnpo/logit": 0.23695942759513855,
|
|
"ht_mnpo/residual": 0.22945943474769592,
|
|
"ht_mnpo/residual_abs": 0.24223342537879944,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -2.222285509109497,
|
|
"logits/rejected": -1.9881519079208374,
|
|
"logps/chosen": -0.3527771532535553,
|
|
"logps/rejected": -0.3299418091773987,
|
|
"loss": 1.1627099514007568,
|
|
"loss/core": 1.1627099514007568,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 3.928004503250122,
|
|
"rewards/margins": 2.369594097137451,
|
|
"rewards/rejected": 1.558410406112671,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6689362718748133,
|
|
"grad_norm": 5.03125,
|
|
"ht_mnpo/logit": 0.14287082850933075,
|
|
"ht_mnpo/residual": 0.13537083566188812,
|
|
"ht_mnpo/residual_abs": 0.21414688229560852,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -2.14042329788208,
|
|
"logits/rejected": -1.9120944738388062,
|
|
"logps/chosen": -0.3552348017692566,
|
|
"logps/rejected": -0.263727605342865,
|
|
"loss": 0.5581281781196594,
|
|
"loss/core": 0.5581281781196594,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 3.3907947540283203,
|
|
"rewards/margins": 1.4287083148956299,
|
|
"rewards/rejected": 1.9620863199234009,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6737143881024906,
|
|
"grad_norm": 0.734375,
|
|
"ht_mnpo/logit": 0.000579959130845964,
|
|
"ht_mnpo/residual": -0.006920039653778076,
|
|
"ht_mnpo/residual_abs": 0.12014114856719971,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -2.2957048416137695,
|
|
"logits/rejected": -1.9869102239608765,
|
|
"logps/chosen": -0.2796603739261627,
|
|
"logps/rejected": -0.29228758811950684,
|
|
"loss": 0.1334332525730133,
|
|
"loss/core": 0.1334332525730133,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.6085796356201172,
|
|
"rewards/margins": 0.005799674894660711,
|
|
"rewards/rejected": 1.602779746055603,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6784925043301678,
|
|
"grad_norm": 5.625,
|
|
"ht_mnpo/logit": 0.023095760494470596,
|
|
"ht_mnpo/residual": 0.015595759265124798,
|
|
"ht_mnpo/residual_abs": 0.10114194452762604,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -2.0242340564727783,
|
|
"logits/rejected": -1.878504753112793,
|
|
"logps/chosen": -0.3086937963962555,
|
|
"logps/rejected": -0.32835036516189575,
|
|
"loss": 0.0820605680346489,
|
|
"loss/core": 0.0820605680346489,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.449677586555481,
|
|
"rewards/margins": 0.23095755279064178,
|
|
"rewards/rejected": 1.2187201976776123,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.683270620557845,
|
|
"grad_norm": 81.5,
|
|
"ht_mnpo/logit": 0.0778246745467186,
|
|
"ht_mnpo/residual": 0.07032467424869537,
|
|
"ht_mnpo/residual_abs": 0.15270543098449707,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -2.0666940212249756,
|
|
"logits/rejected": -1.8228778839111328,
|
|
"logps/chosen": -0.3078117072582245,
|
|
"logps/rejected": -0.3023551106452942,
|
|
"loss": 0.18929661810398102,
|
|
"loss/core": 0.18929661810398102,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 2.2512452602386475,
|
|
"rewards/margins": 0.7782467007637024,
|
|
"rewards/rejected": 1.4729986190795898,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6880487367855223,
|
|
"grad_norm": 3.859375,
|
|
"ht_mnpo/logit": 0.05400656536221504,
|
|
"ht_mnpo/residual": 0.04650656133890152,
|
|
"ht_mnpo/residual_abs": 0.07696235924959183,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -2.173678159713745,
|
|
"logits/rejected": -1.9528112411499023,
|
|
"logps/chosen": -0.2821721136569977,
|
|
"logps/rejected": -0.3137813210487366,
|
|
"loss": 0.03128375485539436,
|
|
"loss/core": 0.03128375485539436,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.233147144317627,
|
|
"rewards/margins": 0.5400656461715698,
|
|
"rewards/rejected": 1.6930816173553467,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6928268530131996,
|
|
"grad_norm": 147.0,
|
|
"ht_mnpo/logit": 0.03776507079601288,
|
|
"ht_mnpo/residual": 0.030265068635344505,
|
|
"ht_mnpo/residual_abs": 0.1769125610589981,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -2.2530579566955566,
|
|
"logits/rejected": -1.901747465133667,
|
|
"logps/chosen": -0.29231947660446167,
|
|
"logps/rejected": -0.30032289028167725,
|
|
"loss": 0.28970783948898315,
|
|
"loss/core": 0.28970783948898315,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.0136709213256836,
|
|
"rewards/margins": 0.37765076756477356,
|
|
"rewards/rejected": 1.6360204219818115,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6976049692408768,
|
|
"grad_norm": 16.375,
|
|
"ht_mnpo/logit": 0.1292092353105545,
|
|
"ht_mnpo/residual": 0.12170922756195068,
|
|
"ht_mnpo/residual_abs": 0.19856879115104675,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.076169967651367,
|
|
"logits/rejected": -1.8429524898529053,
|
|
"logps/chosen": -0.2840667963027954,
|
|
"logps/rejected": -0.30480578541755676,
|
|
"loss": 0.28391191363334656,
|
|
"loss/core": 0.28391191363334656,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.3225340843200684,
|
|
"rewards/margins": 1.2920923233032227,
|
|
"rewards/rejected": 2.0304417610168457,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.702383085468554,
|
|
"grad_norm": 158.0,
|
|
"ht_mnpo/logit": 0.004397551529109478,
|
|
"ht_mnpo/residual": -0.0031024485360831022,
|
|
"ht_mnpo/residual_abs": 0.16906771063804626,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -2.103620767593384,
|
|
"logits/rejected": -1.879486083984375,
|
|
"logps/chosen": -0.314971923828125,
|
|
"logps/rejected": -0.2881259024143219,
|
|
"loss": 0.4299296736717224,
|
|
"loss/core": 0.4299296736717224,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.3823599815368652,
|
|
"rewards/margins": 0.04397547245025635,
|
|
"rewards/rejected": 2.3383851051330566,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.7071612016962313,
|
|
"grad_norm": 2.171875,
|
|
"ht_mnpo/logit": 0.10530763864517212,
|
|
"ht_mnpo/residual": 0.09780765324831009,
|
|
"ht_mnpo/residual_abs": 0.16430619359016418,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -2.098759889602661,
|
|
"logits/rejected": -1.9267299175262451,
|
|
"logps/chosen": -0.27479445934295654,
|
|
"logps/rejected": -0.27386826276779175,
|
|
"loss": 0.2672664225101471,
|
|
"loss/core": 0.2672664225101471,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1735119819641113,
|
|
"rewards/margins": 1.0530763864517212,
|
|
"rewards/rejected": 1.1204354763031006,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7119393179239085,
|
|
"grad_norm": 15.4375,
|
|
"ht_mnpo/logit": 0.09088204801082611,
|
|
"ht_mnpo/residual": 0.08338205516338348,
|
|
"ht_mnpo/residual_abs": 0.10531797260046005,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -2.1337571144104004,
|
|
"logits/rejected": -1.9622604846954346,
|
|
"logps/chosen": -0.28831762075424194,
|
|
"logps/rejected": -0.2993650436401367,
|
|
"loss": 0.07868538051843643,
|
|
"loss/core": 0.07868538051843643,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.8238359689712524,
|
|
"rewards/margins": 0.9088205099105835,
|
|
"rewards/rejected": 0.9150153398513794,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7167174341515857,
|
|
"grad_norm": 51.75,
|
|
"ht_mnpo/logit": 0.05060400813817978,
|
|
"ht_mnpo/residual": 0.043104007840156555,
|
|
"ht_mnpo/residual_abs": 0.06728905439376831,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.316929578781128,
|
|
"logits/rejected": -2.122931957244873,
|
|
"logps/chosen": -0.29240328073501587,
|
|
"logps/rejected": -0.29196321964263916,
|
|
"loss": 0.03344697132706642,
|
|
"loss/core": 0.03344697132706642,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.3464288711547852,
|
|
"rewards/margins": 0.506040096282959,
|
|
"rewards/rejected": 0.8403886556625366,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.7214955503792629,
|
|
"grad_norm": 7.78125,
|
|
"ht_mnpo/logit": 0.11179667711257935,
|
|
"ht_mnpo/residual": 0.10429668426513672,
|
|
"ht_mnpo/residual_abs": 0.17980945110321045,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -2.128986358642578,
|
|
"logits/rejected": -1.8619842529296875,
|
|
"logps/chosen": -0.29354777932167053,
|
|
"logps/rejected": -0.31483781337738037,
|
|
"loss": 0.3539460003376007,
|
|
"loss/core": 0.3539460003376007,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.8777692317962646,
|
|
"rewards/margins": 1.117966651916504,
|
|
"rewards/rejected": 1.7598024606704712,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7262736666069403,
|
|
"grad_norm": 18.625,
|
|
"ht_mnpo/logit": 0.0071696205995976925,
|
|
"ht_mnpo/residual": -0.0003303714038338512,
|
|
"ht_mnpo/residual_abs": 0.1921873241662979,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -2.150174379348755,
|
|
"logits/rejected": -1.9791088104248047,
|
|
"logps/chosen": -0.29564157128334045,
|
|
"logps/rejected": -0.3015114367008209,
|
|
"loss": 0.293133944272995,
|
|
"loss/core": 0.293133944272995,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.4021666049957275,
|
|
"rewards/margins": 0.0716962069272995,
|
|
"rewards/rejected": 2.330470561981201,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7310517828346175,
|
|
"grad_norm": 13.0,
|
|
"ht_mnpo/logit": 0.04746823385357857,
|
|
"ht_mnpo/residual": 0.03996824100613594,
|
|
"ht_mnpo/residual_abs": 0.0960867777466774,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -2.1956920623779297,
|
|
"logits/rejected": -1.8496367931365967,
|
|
"logps/chosen": -0.2813601791858673,
|
|
"logps/rejected": -0.2744866907596588,
|
|
"loss": 0.05206139758229256,
|
|
"loss/core": 0.05206139758229256,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.0784502029418945,
|
|
"rewards/margins": 0.47468239068984985,
|
|
"rewards/rejected": 1.6037676334381104,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7358298990622947,
|
|
"grad_norm": 87.0,
|
|
"ht_mnpo/logit": 0.019193410873413086,
|
|
"ht_mnpo/residual": 0.011693410575389862,
|
|
"ht_mnpo/residual_abs": 0.11662435531616211,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -2.220792293548584,
|
|
"logits/rejected": -2.0077321529388428,
|
|
"logps/chosen": -0.27943068742752075,
|
|
"logps/rejected": -0.2912485897541046,
|
|
"loss": 0.0868755504488945,
|
|
"loss/core": 0.0868755504488945,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.315573215484619,
|
|
"rewards/margins": 0.19193415343761444,
|
|
"rewards/rejected": 2.1236391067504883,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7406080152899719,
|
|
"grad_norm": 3.46875,
|
|
"ht_mnpo/logit": 0.03249342739582062,
|
|
"ht_mnpo/residual": 0.024993427097797394,
|
|
"ht_mnpo/residual_abs": 0.058049798011779785,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -2.2310421466827393,
|
|
"logits/rejected": -2.000556230545044,
|
|
"logps/chosen": -0.3053019642829895,
|
|
"logps/rejected": -0.3055349886417389,
|
|
"loss": 0.022902490571141243,
|
|
"loss/core": 0.022902490571141243,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.5678751468658447,
|
|
"rewards/margins": 0.3249342441558838,
|
|
"rewards/rejected": 1.242940902709961,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7453861315176492,
|
|
"grad_norm": 1.9609375,
|
|
"ht_mnpo/logit": -0.004409261513501406,
|
|
"ht_mnpo/residual": -0.011909263208508492,
|
|
"ht_mnpo/residual_abs": 0.09732024371623993,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.4477899074554443,
|
|
"logits/rejected": -2.261199951171875,
|
|
"logps/chosen": -0.316506028175354,
|
|
"logps/rejected": -0.31703802943229675,
|
|
"loss": 0.20356547832489014,
|
|
"loss/core": 0.20356547832489014,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 0.9836400747299194,
|
|
"rewards/margins": -0.04409266263246536,
|
|
"rewards/rejected": 1.0277327299118042,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7501642477453264,
|
|
"grad_norm": 0.17578125,
|
|
"ht_mnpo/logit": 0.08832958340644836,
|
|
"ht_mnpo/residual": 0.08082958310842514,
|
|
"ht_mnpo/residual_abs": 0.12986600399017334,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -2.177534341812134,
|
|
"logits/rejected": -1.9233522415161133,
|
|
"logps/chosen": -0.29545319080352783,
|
|
"logps/rejected": -0.313882976770401,
|
|
"loss": 0.19263024628162384,
|
|
"loss/core": 0.19263024628162384,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.1445488929748535,
|
|
"rewards/margins": 0.8832958340644836,
|
|
"rewards/rejected": 1.2612531185150146,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7549423639730036,
|
|
"grad_norm": 9.0,
|
|
"ht_mnpo/logit": 0.0722842663526535,
|
|
"ht_mnpo/residual": 0.06478426605463028,
|
|
"ht_mnpo/residual_abs": 0.0955527275800705,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -2.3180410861968994,
|
|
"logits/rejected": -2.018643379211426,
|
|
"logps/chosen": -0.24670171737670898,
|
|
"logps/rejected": -0.25532251596450806,
|
|
"loss": 0.05465935915708542,
|
|
"loss/core": 0.05465935915708542,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9452526569366455,
|
|
"rewards/margins": 0.7228427529335022,
|
|
"rewards/rejected": 1.2224102020263672,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7597204802006808,
|
|
"grad_norm": 159.0,
|
|
"ht_mnpo/logit": -0.06759858131408691,
|
|
"ht_mnpo/residual": -0.07509857416152954,
|
|
"ht_mnpo/residual_abs": 0.16478586196899414,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -2.2069625854492188,
|
|
"logits/rejected": -1.985700249671936,
|
|
"logps/chosen": -0.2841342091560364,
|
|
"logps/rejected": -0.28607645630836487,
|
|
"loss": 0.44809237122535706,
|
|
"loss/core": 0.44809237122535706,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.9460901021957397,
|
|
"rewards/margins": -0.6759856939315796,
|
|
"rewards/rejected": 2.6220760345458984,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7644985964283582,
|
|
"grad_norm": 0.19921875,
|
|
"ht_mnpo/logit": 0.15688112378120422,
|
|
"ht_mnpo/residual": 0.1493811309337616,
|
|
"ht_mnpo/residual_abs": 0.15697996318340302,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.2979936599731445,
|
|
"logits/rejected": -2.1380813121795654,
|
|
"logps/chosen": -0.243856281042099,
|
|
"logps/rejected": -0.25572383403778076,
|
|
"loss": 0.35162821412086487,
|
|
"loss/core": 0.35162821412086487,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6728522777557373,
|
|
"rewards/margins": 1.5688111782073975,
|
|
"rewards/rejected": 1.1040410995483398,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7692767126560354,
|
|
"grad_norm": 2.578125,
|
|
"ht_mnpo/logit": 0.023322004824876785,
|
|
"ht_mnpo/residual": 0.01582200825214386,
|
|
"ht_mnpo/residual_abs": 0.059140510857105255,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.3517727851867676,
|
|
"logits/rejected": -2.1378703117370605,
|
|
"logps/chosen": -0.31542864441871643,
|
|
"logps/rejected": -0.3220391571521759,
|
|
"loss": 0.021347444504499435,
|
|
"loss/core": 0.021347444504499435,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 0.9367267489433289,
|
|
"rewards/margins": 0.23322002589702606,
|
|
"rewards/rejected": 0.7035066485404968,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.7740548288837126,
|
|
"grad_norm": 251.0,
|
|
"ht_mnpo/logit": 0.18958637118339539,
|
|
"ht_mnpo/residual": 0.18208637833595276,
|
|
"ht_mnpo/residual_abs": 0.21843548119068146,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.0479319095611572,
|
|
"logits/rejected": -1.8690176010131836,
|
|
"logps/chosen": -0.3233865797519684,
|
|
"logps/rejected": -0.330311119556427,
|
|
"loss": 0.7746216654777527,
|
|
"loss/core": 0.7746216654777527,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.8327841758728027,
|
|
"rewards/margins": 1.8958637714385986,
|
|
"rewards/rejected": 0.9369203448295593,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7788329451113898,
|
|
"grad_norm": 62.5,
|
|
"ht_mnpo/logit": 0.08295156061649323,
|
|
"ht_mnpo/residual": 0.0754515528678894,
|
|
"ht_mnpo/residual_abs": 0.1541987508535385,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -2.2893285751342773,
|
|
"logits/rejected": -2.0092718601226807,
|
|
"logps/chosen": -0.2440989464521408,
|
|
"logps/rejected": -0.2622814476490021,
|
|
"loss": 0.1557014435529709,
|
|
"loss/core": 0.1557014435529709,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.43802547454834,
|
|
"rewards/margins": 0.8295154571533203,
|
|
"rewards/rejected": 1.6085097789764404,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7836110613390671,
|
|
"grad_norm": 6.78125,
|
|
"ht_mnpo/logit": 0.14719262719154358,
|
|
"ht_mnpo/residual": 0.13969263434410095,
|
|
"ht_mnpo/residual_abs": 0.17651373147964478,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -2.12347412109375,
|
|
"logits/rejected": -1.8785676956176758,
|
|
"logps/chosen": -0.27428001165390015,
|
|
"logps/rejected": -0.3116721212863922,
|
|
"loss": 0.4936157763004303,
|
|
"loss/core": 0.4936157763004303,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.819645643234253,
|
|
"rewards/margins": 1.4719264507293701,
|
|
"rewards/rejected": 1.3477191925048828,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.7883891775667443,
|
|
"grad_norm": 736.0,
|
|
"ht_mnpo/logit": 0.2026674449443817,
|
|
"ht_mnpo/residual": 0.1951674520969391,
|
|
"ht_mnpo/residual_abs": 0.22531075775623322,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.2073588371276855,
|
|
"logits/rejected": -2.0113062858581543,
|
|
"logps/chosen": -0.3016754984855652,
|
|
"logps/rejected": -0.29153111577033997,
|
|
"loss": 0.9155440330505371,
|
|
"loss/core": 0.9155440330505371,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 2.9878766536712646,
|
|
"rewards/margins": 2.026674270629883,
|
|
"rewards/rejected": 0.9612022638320923,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7931672937944215,
|
|
"grad_norm": 0.83203125,
|
|
"ht_mnpo/logit": 0.06144893914461136,
|
|
"ht_mnpo/residual": 0.053948938846588135,
|
|
"ht_mnpo/residual_abs": 0.08334354311227798,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.2769930362701416,
|
|
"logits/rejected": -2.078378200531006,
|
|
"logps/chosen": -0.28695565462112427,
|
|
"logps/rejected": -0.2857491075992584,
|
|
"loss": 0.08329080790281296,
|
|
"loss/core": 0.08329080790281296,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 1.6761438846588135,
|
|
"rewards/margins": 0.6144893169403076,
|
|
"rewards/rejected": 1.0616545677185059,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7979454100220987,
|
|
"grad_norm": 428.0,
|
|
"ht_mnpo/logit": -0.0352485366165638,
|
|
"ht_mnpo/residual": -0.04274853318929672,
|
|
"ht_mnpo/residual_abs": 0.21524009108543396,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -2.257803440093994,
|
|
"logits/rejected": -1.8916332721710205,
|
|
"logps/chosen": -0.280212938785553,
|
|
"logps/rejected": -0.34277522563934326,
|
|
"loss": 0.42016157507896423,
|
|
"loss/core": 0.42016157507896423,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.795287609100342,
|
|
"rewards/margins": -0.3524852693080902,
|
|
"rewards/rejected": 3.147773027420044,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.802723526249776,
|
|
"grad_norm": 10.375,
|
|
"ht_mnpo/logit": 0.022869501262903214,
|
|
"ht_mnpo/residual": 0.015369502827525139,
|
|
"ht_mnpo/residual_abs": 0.06976980715990067,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.278902530670166,
|
|
"logits/rejected": -2.0701346397399902,
|
|
"logps/chosen": -0.2895943224430084,
|
|
"logps/rejected": -0.31108802556991577,
|
|
"loss": 0.021526243537664413,
|
|
"loss/core": 0.021526243537664413,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.143057107925415,
|
|
"rewards/margins": 0.22869500517845154,
|
|
"rewards/rejected": 0.9143620729446411,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8075016424774533,
|
|
"grad_norm": 4.71875,
|
|
"ht_mnpo/logit": 0.03050408326089382,
|
|
"ht_mnpo/residual": 0.0230040792375803,
|
|
"ht_mnpo/residual_abs": 0.16995054483413696,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -2.1468443870544434,
|
|
"logits/rejected": -1.917742133140564,
|
|
"logps/chosen": -0.2518450617790222,
|
|
"logps/rejected": -0.27282077074050903,
|
|
"loss": 0.4268716275691986,
|
|
"loss/core": 0.4268716275691986,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2956597805023193,
|
|
"rewards/margins": 0.3050408363342285,
|
|
"rewards/rejected": 1.9906187057495117,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8122797587051305,
|
|
"grad_norm": 2.1875,
|
|
"ht_mnpo/logit": 0.01607021503150463,
|
|
"ht_mnpo/residual": 0.008570205420255661,
|
|
"ht_mnpo/residual_abs": 0.14475668966770172,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -2.292693614959717,
|
|
"logits/rejected": -2.110647678375244,
|
|
"logps/chosen": -0.3145769536495209,
|
|
"logps/rejected": -0.30854150652885437,
|
|
"loss": 0.40554890036582947,
|
|
"loss/core": 0.40554890036582947,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.2903105020523071,
|
|
"rewards/margins": 0.16070207953453064,
|
|
"rewards/rejected": 1.129608392715454,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8170578749328078,
|
|
"grad_norm": 169.0,
|
|
"ht_mnpo/logit": 0.0835026353597641,
|
|
"ht_mnpo/residual": 0.07600263506174088,
|
|
"ht_mnpo/residual_abs": 0.19562701880931854,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -2.2144742012023926,
|
|
"logits/rejected": -1.9130665063858032,
|
|
"logps/chosen": -0.2866402268409729,
|
|
"logps/rejected": -0.29126277565956116,
|
|
"loss": 0.24363788962364197,
|
|
"loss/core": 0.24363788962364197,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.9844703674316406,
|
|
"rewards/margins": 0.8350263833999634,
|
|
"rewards/rejected": 2.149444103240967,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.821835991160485,
|
|
"grad_norm": 89.5,
|
|
"ht_mnpo/logit": 0.12511678040027618,
|
|
"ht_mnpo/residual": 0.11761675775051117,
|
|
"ht_mnpo/residual_abs": 0.19286657869815826,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -2.188568592071533,
|
|
"logits/rejected": -1.984760046005249,
|
|
"logps/chosen": -0.3057268261909485,
|
|
"logps/rejected": -0.3176608979701996,
|
|
"loss": 0.2807512879371643,
|
|
"loss/core": 0.2807512879371643,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.978262424468994,
|
|
"rewards/margins": 1.2511677742004395,
|
|
"rewards/rejected": 1.7270948886871338,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.8266141073881622,
|
|
"grad_norm": 158.0,
|
|
"ht_mnpo/logit": 0.01128818467259407,
|
|
"ht_mnpo/residual": 0.00378818204626441,
|
|
"ht_mnpo/residual_abs": 0.15559642016887665,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -2.1104423999786377,
|
|
"logits/rejected": -1.8544162511825562,
|
|
"logps/chosen": -0.27419915795326233,
|
|
"logps/rejected": -0.3105354309082031,
|
|
"loss": 0.20748119056224823,
|
|
"loss/core": 0.20748119056224823,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2308783531188965,
|
|
"rewards/margins": 0.11288192123174667,
|
|
"rewards/rejected": 2.1179964542388916,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8313922236158394,
|
|
"grad_norm": 2.625,
|
|
"ht_mnpo/logit": 0.07102836668491364,
|
|
"ht_mnpo/residual": 0.06352836638689041,
|
|
"ht_mnpo/residual_abs": 0.13127169013023376,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.0416035652160645,
|
|
"logits/rejected": -1.8697378635406494,
|
|
"logps/chosen": -0.27052196860313416,
|
|
"logps/rejected": -0.290427029132843,
|
|
"loss": 0.16899879276752472,
|
|
"loss/core": 0.16899879276752472,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2697510719299316,
|
|
"rewards/margins": 0.7102835774421692,
|
|
"rewards/rejected": 1.5594675540924072,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8361703398435167,
|
|
"grad_norm": 76.5,
|
|
"ht_mnpo/logit": -0.0015212118159979582,
|
|
"ht_mnpo/residual": -0.009021207690238953,
|
|
"ht_mnpo/residual_abs": 0.08979154378175735,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -2.4572672843933105,
|
|
"logits/rejected": -2.1243271827697754,
|
|
"logps/chosen": -0.2736932635307312,
|
|
"logps/rejected": -0.296703040599823,
|
|
"loss": 0.06740878522396088,
|
|
"loss/core": 0.06740878522396088,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.2251904010772705,
|
|
"rewards/margins": -0.0152121065184474,
|
|
"rewards/rejected": 1.2404022216796875,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840948456071194,
|
|
"grad_norm": 2.078125,
|
|
"ht_mnpo/logit": 0.00960737094283104,
|
|
"ht_mnpo/residual": 0.0021073739044368267,
|
|
"ht_mnpo/residual_abs": 0.08030150830745697,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.3116345405578613,
|
|
"logits/rejected": -2.1608469486236572,
|
|
"logps/chosen": -0.28452223539352417,
|
|
"logps/rejected": -0.30294761061668396,
|
|
"loss": 0.09350978583097458,
|
|
"loss/core": 0.09350978583097458,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2112499475479126,
|
|
"rewards/margins": 0.0960737094283104,
|
|
"rewards/rejected": 1.1151762008666992,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8457265722988712,
|
|
"grad_norm": 7.125,
|
|
"ht_mnpo/logit": 0.09174073487520218,
|
|
"ht_mnpo/residual": 0.08424073457717896,
|
|
"ht_mnpo/residual_abs": 0.13746334612369537,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -2.1234521865844727,
|
|
"logits/rejected": -1.9998477697372437,
|
|
"logps/chosen": -0.2799425721168518,
|
|
"logps/rejected": -0.2617390751838684,
|
|
"loss": 0.14151671528816223,
|
|
"loss/core": 0.14151671528816223,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.404963970184326,
|
|
"rewards/margins": 0.9174073934555054,
|
|
"rewards/rejected": 1.4875565767288208,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8505046885265484,
|
|
"grad_norm": 19.625,
|
|
"ht_mnpo/logit": 0.09642306715250015,
|
|
"ht_mnpo/residual": 0.08892306685447693,
|
|
"ht_mnpo/residual_abs": 0.11294756829738617,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -2.3085834980010986,
|
|
"logits/rejected": -2.0683159828186035,
|
|
"logps/chosen": -0.2952830195426941,
|
|
"logps/rejected": -0.30370476841926575,
|
|
"loss": 0.17521965503692627,
|
|
"loss/core": 0.17521965503692627,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.697967290878296,
|
|
"rewards/margins": 0.9642306566238403,
|
|
"rewards/rejected": 1.7337367534637451,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.8552828047542257,
|
|
"grad_norm": 16.625,
|
|
"ht_mnpo/logit": 0.05871545523405075,
|
|
"ht_mnpo/residual": 0.05121545121073723,
|
|
"ht_mnpo/residual_abs": 0.1308048516511917,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -2.4693121910095215,
|
|
"logits/rejected": -2.181762218475342,
|
|
"logps/chosen": -0.2554033398628235,
|
|
"logps/rejected": -0.2718905806541443,
|
|
"loss": 0.20783178508281708,
|
|
"loss/core": 0.20783178508281708,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.2181339263916016,
|
|
"rewards/margins": 0.5871545672416687,
|
|
"rewards/rejected": 1.6309795379638672,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8600609209819029,
|
|
"grad_norm": 97.0,
|
|
"ht_mnpo/logit": 0.06254192441701889,
|
|
"ht_mnpo/residual": 0.05504193156957626,
|
|
"ht_mnpo/residual_abs": 0.15723498165607452,
|
|
"ht_mnpo/target": 0.007499999366700649,
|
|
"ht_mnpo/target_abs": 0.007499999366700649,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -2.3032777309417725,
|
|
"logits/rejected": -2.058039665222168,
|
|
"logps/chosen": -0.27136459946632385,
|
|
"logps/rejected": -0.27709969878196716,
|
|
"loss": 0.19733476638793945,
|
|
"loss/core": 0.19733476638793945,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.982447862625122,
|
|
"rewards/margins": 0.6254192590713501,
|
|
"rewards/rejected": 1.3570287227630615,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8600609209819029,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.3149479303053684,
|
|
"train_runtime": 7160.3559,
|
|
"train_samples_per_second": 2.011,
|
|
"train_steps_per_second": 0.126
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|