2200 lines
86 KiB
JSON
2200 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5234231876472127,
|
|
"eval_steps": 500,
|
|
"global_step": 1000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0005234231876472127,
|
|
"grad_norm": 126.29230499267578,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.40118408203125,
|
|
"logits/rejected": -0.41802978515625,
|
|
"logps/chosen": -297.609375,
|
|
"logps/rejected": -247.84375,
|
|
"logps/weighted_chosen": -4.5152587890625,
|
|
"logps/weighted_rejected": -3.032470703125,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"rewards/weighted_accuracies": 0.0,
|
|
"rewards/weighted_chosen": 0.0,
|
|
"rewards/weighted_margins": 0.0,
|
|
"rewards/weighted_rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.005234231876472127,
|
|
"grad_norm": 296.4369812011719,
|
|
"learning_rate": 4.6875e-08,
|
|
"logits/chosen": -0.3177456259727478,
|
|
"logits/rejected": -0.3534359335899353,
|
|
"logps/chosen": -275.5711669921875,
|
|
"logps/rejected": -255.90451049804688,
|
|
"logps/weighted_chosen": -2.350965738296509,
|
|
"logps/weighted_rejected": -2.549940347671509,
|
|
"loss": 0.6917,
|
|
"rewards/accuracies": 0.25,
|
|
"rewards/chosen": -0.0401475690305233,
|
|
"rewards/margins": 0.04296875,
|
|
"rewards/rejected": -0.0831163227558136,
|
|
"rewards/weighted_accuracies": 0.3229166567325592,
|
|
"rewards/weighted_chosen": -0.00032212998485192657,
|
|
"rewards/weighted_margins": 0.00019327799964230508,
|
|
"rewards/weighted_rejected": -0.0005154079990461469,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.010468463752944255,
|
|
"grad_norm": 21.67967414855957,
|
|
"learning_rate": 9.895833333333332e-08,
|
|
"logits/chosen": -0.29769057035446167,
|
|
"logits/rejected": -0.313650518655777,
|
|
"logps/chosen": -294.3374938964844,
|
|
"logps/rejected": -272.6703186035156,
|
|
"logps/weighted_chosen": -2.13031005859375,
|
|
"logps/weighted_rejected": -2.103222608566284,
|
|
"loss": 0.6908,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": -0.06103515625,
|
|
"rewards/margins": -0.01318359375,
|
|
"rewards/rejected": -0.0478515625,
|
|
"rewards/weighted_accuracies": 0.4437499940395355,
|
|
"rewards/weighted_chosen": 0.0014366150135174394,
|
|
"rewards/weighted_margins": 0.0021545409690588713,
|
|
"rewards/weighted_rejected": -0.0007179260137490928,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.015702695629416383,
|
|
"grad_norm": 76.9887466430664,
|
|
"learning_rate": 1.5104166666666664e-07,
|
|
"logits/chosen": -0.2917121946811676,
|
|
"logits/rejected": -0.337240606546402,
|
|
"logps/chosen": -298.02655029296875,
|
|
"logps/rejected": -268.12188720703125,
|
|
"logps/weighted_chosen": -2.0724120140075684,
|
|
"logps/weighted_rejected": -2.4466919898986816,
|
|
"loss": 0.6912,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": -0.0062500000931322575,
|
|
"rewards/margins": -0.02509765699505806,
|
|
"rewards/rejected": 0.01884765550494194,
|
|
"rewards/weighted_accuracies": 0.4281249940395355,
|
|
"rewards/weighted_chosen": 0.0027938843704760075,
|
|
"rewards/weighted_margins": 0.0019706725142896175,
|
|
"rewards/weighted_rejected": 0.0008232116815634072,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.02093692750588851,
|
|
"grad_norm": 32.98203659057617,
|
|
"learning_rate": 2.03125e-07,
|
|
"logits/chosen": -0.3011154234409332,
|
|
"logits/rejected": -0.3432762026786804,
|
|
"logps/chosen": -278.63751220703125,
|
|
"logps/rejected": -253.88125610351562,
|
|
"logps/weighted_chosen": -2.2070555686950684,
|
|
"logps/weighted_rejected": -2.605224609375,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.26249998807907104,
|
|
"rewards/chosen": 0.0034667968284338713,
|
|
"rewards/margins": -0.05991210788488388,
|
|
"rewards/rejected": 0.06337890774011612,
|
|
"rewards/weighted_accuracies": 0.35624998807907104,
|
|
"rewards/weighted_chosen": 0.0014549255138263106,
|
|
"rewards/weighted_margins": -0.00034332275390625,
|
|
"rewards/weighted_rejected": 0.0017982482677325606,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.02617115938236064,
|
|
"grad_norm": 20.751684188842773,
|
|
"learning_rate": 2.552083333333333e-07,
|
|
"logits/chosen": -0.2822524905204773,
|
|
"logits/rejected": -0.32080918550491333,
|
|
"logps/chosen": -280.31329345703125,
|
|
"logps/rejected": -267.58709716796875,
|
|
"logps/weighted_chosen": -2.136962890625,
|
|
"logps/weighted_rejected": -2.1753907203674316,
|
|
"loss": 0.6883,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": -0.07236327975988388,
|
|
"rewards/margins": -0.09189452975988388,
|
|
"rewards/rejected": 0.01953125,
|
|
"rewards/weighted_accuracies": 0.4375,
|
|
"rewards/weighted_chosen": 0.0054107666946947575,
|
|
"rewards/weighted_margins": 0.0078063965775072575,
|
|
"rewards/weighted_rejected": -0.0023956298828125,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.031405391258832765,
|
|
"grad_norm": 40.70024108886719,
|
|
"learning_rate": 3.0729166666666665e-07,
|
|
"logits/chosen": -0.3149581849575043,
|
|
"logits/rejected": -0.3086872100830078,
|
|
"logps/chosen": -277.6031188964844,
|
|
"logps/rejected": -261.8031311035156,
|
|
"logps/weighted_chosen": -2.5905518531799316,
|
|
"logps/weighted_rejected": -2.4834961891174316,
|
|
"loss": 0.6874,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": 0.03662109375,
|
|
"rewards/margins": 0.12646484375,
|
|
"rewards/rejected": -0.08984375,
|
|
"rewards/weighted_accuracies": 0.5,
|
|
"rewards/weighted_chosen": 0.0004280090215615928,
|
|
"rewards/weighted_margins": 0.01105651818215847,
|
|
"rewards/weighted_rejected": -0.01062927208840847,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.036639623135304895,
|
|
"grad_norm": 67.51947021484375,
|
|
"learning_rate": 3.59375e-07,
|
|
"logits/chosen": -0.318746954202652,
|
|
"logits/rejected": -0.32574766874313354,
|
|
"logps/chosen": -289.90313720703125,
|
|
"logps/rejected": -245.04452514648438,
|
|
"logps/weighted_chosen": -2.098431348800659,
|
|
"logps/weighted_rejected": -2.392407178878784,
|
|
"loss": 0.6841,
|
|
"rewards/accuracies": 0.48750001192092896,
|
|
"rewards/chosen": 0.16708984971046448,
|
|
"rewards/margins": 0.4442382752895355,
|
|
"rewards/rejected": -0.27714842557907104,
|
|
"rewards/weighted_accuracies": 0.559374988079071,
|
|
"rewards/weighted_chosen": 0.015575408935546875,
|
|
"rewards/weighted_margins": 0.02174072340130806,
|
|
"rewards/weighted_rejected": -0.00616531353443861,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.04187385501177702,
|
|
"grad_norm": 68.87100982666016,
|
|
"learning_rate": 4.114583333333333e-07,
|
|
"logits/chosen": -0.286581426858902,
|
|
"logits/rejected": -0.3082527220249176,
|
|
"logps/chosen": -289.5101623535156,
|
|
"logps/rejected": -270.4375,
|
|
"logps/weighted_chosen": -2.2385497093200684,
|
|
"logps/weighted_rejected": -2.4218382835388184,
|
|
"loss": 0.6727,
|
|
"rewards/accuracies": 0.4749999940395355,
|
|
"rewards/chosen": 0.31572264432907104,
|
|
"rewards/margins": 0.5547851324081421,
|
|
"rewards/rejected": -0.23906250298023224,
|
|
"rewards/weighted_accuracies": 0.596875011920929,
|
|
"rewards/weighted_chosen": 0.03613891452550888,
|
|
"rewards/weighted_margins": 0.05283202975988388,
|
|
"rewards/weighted_rejected": -0.01669769361615181,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.04710808688824915,
|
|
"grad_norm": 40.29203414916992,
|
|
"learning_rate": 4.6354166666666664e-07,
|
|
"logits/chosen": -0.3158706724643707,
|
|
"logits/rejected": -0.30914992094039917,
|
|
"logps/chosen": -280.5726623535156,
|
|
"logps/rejected": -258.17657470703125,
|
|
"logps/weighted_chosen": -2.45281982421875,
|
|
"logps/weighted_rejected": -2.5444703102111816,
|
|
"loss": 0.6683,
|
|
"rewards/accuracies": 0.4906249940395355,
|
|
"rewards/chosen": 0.15966796875,
|
|
"rewards/margins": 0.599609375,
|
|
"rewards/rejected": -0.43994140625,
|
|
"rewards/weighted_accuracies": 0.581250011920929,
|
|
"rewards/weighted_chosen": 0.05808715894818306,
|
|
"rewards/weighted_margins": 0.07471618801355362,
|
|
"rewards/weighted_rejected": -0.0166168212890625,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.05234231876472128,
|
|
"grad_norm": 46.855377197265625,
|
|
"learning_rate": 5.156249999999999e-07,
|
|
"logits/chosen": -0.2856552004814148,
|
|
"logits/rejected": -0.3585342466831207,
|
|
"logps/chosen": -291.05548095703125,
|
|
"logps/rejected": -287.078125,
|
|
"logps/weighted_chosen": -1.9577789306640625,
|
|
"logps/weighted_rejected": -2.532482862472534,
|
|
"loss": 0.6785,
|
|
"rewards/accuracies": 0.5625,
|
|
"rewards/chosen": -0.13925781846046448,
|
|
"rewards/margins": 0.9869140386581421,
|
|
"rewards/rejected": -1.1261718273162842,
|
|
"rewards/weighted_accuracies": 0.590624988079071,
|
|
"rewards/weighted_chosen": 0.03715210035443306,
|
|
"rewards/weighted_margins": 0.0635833740234375,
|
|
"rewards/weighted_rejected": -0.02643737755715847,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.05757655064119341,
|
|
"grad_norm": 55.04579162597656,
|
|
"learning_rate": 5.677083333333333e-07,
|
|
"logits/chosen": -0.33493995666503906,
|
|
"logits/rejected": -0.3254844546318054,
|
|
"logps/chosen": -297.2953186035156,
|
|
"logps/rejected": -262.6773376464844,
|
|
"logps/weighted_chosen": -2.606689453125,
|
|
"logps/weighted_rejected": -2.648364305496216,
|
|
"loss": 0.6821,
|
|
"rewards/accuracies": 0.528124988079071,
|
|
"rewards/chosen": -0.9228515625,
|
|
"rewards/margins": 0.8955078125,
|
|
"rewards/rejected": -1.818359375,
|
|
"rewards/weighted_accuracies": 0.518750011920929,
|
|
"rewards/weighted_chosen": -0.005747986026108265,
|
|
"rewards/weighted_margins": 0.05161895602941513,
|
|
"rewards/weighted_rejected": -0.05732421949505806,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.06281078251766553,
|
|
"grad_norm": 22.23135757446289,
|
|
"learning_rate": 6.197916666666666e-07,
|
|
"logits/chosen": -0.3393222689628601,
|
|
"logits/rejected": -0.36481350660324097,
|
|
"logps/chosen": -295.6703186035156,
|
|
"logps/rejected": -256.3296813964844,
|
|
"logps/weighted_chosen": -1.8351562023162842,
|
|
"logps/weighted_rejected": -2.124218702316284,
|
|
"loss": 0.6752,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -1.071679711341858,
|
|
"rewards/margins": 1.46142578125,
|
|
"rewards/rejected": -2.5331053733825684,
|
|
"rewards/weighted_accuracies": 0.546875,
|
|
"rewards/weighted_chosen": 0.0018810272449627519,
|
|
"rewards/weighted_margins": 0.06835174560546875,
|
|
"rewards/weighted_rejected": -0.0664466843008995,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.06804501439413765,
|
|
"grad_norm": 57.93917465209961,
|
|
"learning_rate": 6.718749999999999e-07,
|
|
"logits/chosen": -0.30284881591796875,
|
|
"logits/rejected": -0.2989334166049957,
|
|
"logps/chosen": -306.5074157714844,
|
|
"logps/rejected": -279.8265686035156,
|
|
"logps/weighted_chosen": -1.910064697265625,
|
|
"logps/weighted_rejected": -2.2278685569763184,
|
|
"loss": 0.6738,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -0.45097655057907104,
|
|
"rewards/margins": 1.7268555164337158,
|
|
"rewards/rejected": -2.177734375,
|
|
"rewards/weighted_accuracies": 0.59375,
|
|
"rewards/weighted_chosen": 0.02166290208697319,
|
|
"rewards/weighted_margins": 0.07758025825023651,
|
|
"rewards/weighted_rejected": -0.05589141696691513,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.07327924627060979,
|
|
"grad_norm": 66.64070892333984,
|
|
"learning_rate": 7.239583333333333e-07,
|
|
"logits/chosen": -0.34190064668655396,
|
|
"logits/rejected": -0.3586837649345398,
|
|
"logps/chosen": -300.01483154296875,
|
|
"logps/rejected": -276.1703186035156,
|
|
"logps/weighted_chosen": -2.202807664871216,
|
|
"logps/weighted_rejected": -2.474353075027466,
|
|
"loss": 0.6635,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -1.3230469226837158,
|
|
"rewards/margins": 2.2220702171325684,
|
|
"rewards/rejected": -3.545117139816284,
|
|
"rewards/weighted_accuracies": 0.609375,
|
|
"rewards/weighted_chosen": 0.0006683349492959678,
|
|
"rewards/weighted_margins": 0.10604552924633026,
|
|
"rewards/weighted_rejected": -0.1053924560546875,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.07851347814708191,
|
|
"grad_norm": 18.789766311645508,
|
|
"learning_rate": 7.760416666666666e-07,
|
|
"logits/chosen": -0.2976974546909332,
|
|
"logits/rejected": -0.3081321716308594,
|
|
"logps/chosen": -286.27813720703125,
|
|
"logps/rejected": -255.4640655517578,
|
|
"logps/weighted_chosen": -2.7657103538513184,
|
|
"logps/weighted_rejected": -2.831347703933716,
|
|
"loss": 0.6605,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -2.104687452316284,
|
|
"rewards/margins": 2.5054688453674316,
|
|
"rewards/rejected": -4.610156059265137,
|
|
"rewards/weighted_accuracies": 0.621874988079071,
|
|
"rewards/weighted_chosen": -0.0018630981212481856,
|
|
"rewards/weighted_margins": 0.158416748046875,
|
|
"rewards/weighted_rejected": -0.1603546142578125,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.08374771002355404,
|
|
"grad_norm": 51.51210021972656,
|
|
"learning_rate": 8.28125e-07,
|
|
"logits/chosen": -0.3341739773750305,
|
|
"logits/rejected": -0.3859619200229645,
|
|
"logps/chosen": -306.4765625,
|
|
"logps/rejected": -279.1148376464844,
|
|
"logps/weighted_chosen": -2.3189454078674316,
|
|
"logps/weighted_rejected": -2.36669921875,
|
|
"loss": 0.636,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -2.575390577316284,
|
|
"rewards/margins": 3.349609375,
|
|
"rewards/rejected": -5.925000190734863,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": 0.02147369459271431,
|
|
"rewards/weighted_margins": 0.22438660264015198,
|
|
"rewards/weighted_rejected": -0.203105166554451,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.08898194190002617,
|
|
"grad_norm": 398.3809509277344,
|
|
"learning_rate": 8.802083333333333e-07,
|
|
"logits/chosen": -0.36855775117874146,
|
|
"logits/rejected": -0.37070387601852417,
|
|
"logps/chosen": -307.1656188964844,
|
|
"logps/rejected": -265.78436279296875,
|
|
"logps/weighted_chosen": -2.459460496902466,
|
|
"logps/weighted_rejected": -2.757373094558716,
|
|
"loss": 0.6811,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -4.097460746765137,
|
|
"rewards/margins": 3.488476514816284,
|
|
"rewards/rejected": -7.585839748382568,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": 0.018505096435546875,
|
|
"rewards/weighted_margins": 0.19701537489891052,
|
|
"rewards/weighted_rejected": -0.17839965224266052,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.0942161737764983,
|
|
"grad_norm": 55.77580261230469,
|
|
"learning_rate": 9.322916666666666e-07,
|
|
"logits/chosen": -0.3392753601074219,
|
|
"logits/rejected": -0.35816192626953125,
|
|
"logps/chosen": -278.99530029296875,
|
|
"logps/rejected": -265.18359375,
|
|
"logps/weighted_chosen": -2.362103223800659,
|
|
"logps/weighted_rejected": -2.754711866378784,
|
|
"loss": 0.6944,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -5.346972465515137,
|
|
"rewards/margins": 3.5015625953674316,
|
|
"rewards/rejected": -8.848730087280273,
|
|
"rewards/weighted_accuracies": 0.578125,
|
|
"rewards/weighted_chosen": -0.05782318115234375,
|
|
"rewards/weighted_margins": 0.16480103135108948,
|
|
"rewards/weighted_rejected": -0.22255554795265198,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.09945040565297043,
|
|
"grad_norm": 38.015960693359375,
|
|
"learning_rate": 9.84375e-07,
|
|
"logits/chosen": -0.3686843812465668,
|
|
"logits/rejected": -0.4041244387626648,
|
|
"logps/chosen": -314.3070373535156,
|
|
"logps/rejected": -276.7484436035156,
|
|
"logps/weighted_chosen": -2.123486280441284,
|
|
"logps/weighted_rejected": -2.6261963844299316,
|
|
"loss": 0.6392,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -8.581738471984863,
|
|
"rewards/margins": 4.317968845367432,
|
|
"rewards/rejected": -12.900781631469727,
|
|
"rewards/weighted_accuracies": 0.621874988079071,
|
|
"rewards/weighted_chosen": -0.129638671875,
|
|
"rewards/weighted_margins": 0.2160186767578125,
|
|
"rewards/weighted_rejected": -0.345590204000473,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.10468463752944256,
|
|
"grad_norm": 46.52367401123047,
|
|
"learning_rate": 9.99959085414323e-07,
|
|
"logits/chosen": -0.4128967225551605,
|
|
"logits/rejected": -0.4471847414970398,
|
|
"logps/chosen": -320.0546875,
|
|
"logps/rejected": -273.11248779296875,
|
|
"logps/weighted_chosen": -2.5019164085388184,
|
|
"logps/weighted_rejected": -2.9936890602111816,
|
|
"loss": 0.6473,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -10.737597465515137,
|
|
"rewards/margins": 4.738671779632568,
|
|
"rewards/rejected": -15.476171493530273,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.11443634331226349,
|
|
"rewards/weighted_margins": 0.2610321044921875,
|
|
"rewards/weighted_rejected": -0.37534791231155396,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.10991886940591468,
|
|
"grad_norm": 21.238189697265625,
|
|
"learning_rate": 9.997587035630105e-07,
|
|
"logits/chosen": -0.4288749694824219,
|
|
"logits/rejected": -0.4688262939453125,
|
|
"logps/chosen": -300.0765686035156,
|
|
"logps/rejected": -304.63751220703125,
|
|
"logps/weighted_chosen": -2.32427978515625,
|
|
"logps/weighted_rejected": -3.0592284202575684,
|
|
"loss": 0.6424,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -13.117578506469727,
|
|
"rewards/margins": 7.013671875,
|
|
"rewards/rejected": -20.133594512939453,
|
|
"rewards/weighted_accuracies": 0.653124988079071,
|
|
"rewards/weighted_chosen": -0.21423491835594177,
|
|
"rewards/weighted_margins": 0.27025145292282104,
|
|
"rewards/weighted_rejected": -0.4845077395439148,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.11515310128238682,
|
|
"grad_norm": 24.92041015625,
|
|
"learning_rate": 9.99391406364405e-07,
|
|
"logits/chosen": -0.42696380615234375,
|
|
"logits/rejected": -0.429006963968277,
|
|
"logps/chosen": -305.4906311035156,
|
|
"logps/rejected": -288.6312561035156,
|
|
"logps/weighted_chosen": -2.625018358230591,
|
|
"logps/weighted_rejected": -3.102160692214966,
|
|
"loss": 0.6601,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -13.349413871765137,
|
|
"rewards/margins": 6.373632907867432,
|
|
"rewards/rejected": -19.72265625,
|
|
"rewards/weighted_accuracies": 0.6156250238418579,
|
|
"rewards/weighted_chosen": -0.20062866806983948,
|
|
"rewards/weighted_margins": 0.316873162984848,
|
|
"rewards/weighted_rejected": -0.5174545049667358,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.12038733315885894,
|
|
"grad_norm": 147.95851135253906,
|
|
"learning_rate": 9.988573164927884e-07,
|
|
"logits/chosen": -0.3811447024345398,
|
|
"logits/rejected": -0.4161086976528168,
|
|
"logps/chosen": -281.33203125,
|
|
"logps/rejected": -274.234375,
|
|
"logps/weighted_chosen": -2.32806396484375,
|
|
"logps/weighted_rejected": -2.6552734375,
|
|
"loss": 0.7195,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -12.46875,
|
|
"rewards/margins": 10.046093940734863,
|
|
"rewards/rejected": -22.515430450439453,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": -0.24639587104320526,
|
|
"rewards/weighted_margins": 0.23908081650733948,
|
|
"rewards/weighted_rejected": -0.4853073060512543,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.12562156503533106,
|
|
"grad_norm": 26.882122039794922,
|
|
"learning_rate": 9.98156612329838e-07,
|
|
"logits/chosen": -0.4748245179653168,
|
|
"logits/rejected": -0.5250595211982727,
|
|
"logps/chosen": -278.16717529296875,
|
|
"logps/rejected": -306.29376220703125,
|
|
"logps/weighted_chosen": -2.348803758621216,
|
|
"logps/weighted_rejected": -2.9455933570861816,
|
|
"loss": 0.6674,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -13.405566215515137,
|
|
"rewards/margins": 10.753710746765137,
|
|
"rewards/rejected": -24.158985137939453,
|
|
"rewards/weighted_accuracies": 0.625,
|
|
"rewards/weighted_chosen": -0.14908751845359802,
|
|
"rewards/weighted_margins": 0.33162689208984375,
|
|
"rewards/weighted_rejected": -0.48021697998046875,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.13085579691180318,
|
|
"grad_norm": 86.49760437011719,
|
|
"learning_rate": 9.97289527905053e-07,
|
|
"logits/chosen": -0.478302001953125,
|
|
"logits/rejected": -0.48861923813819885,
|
|
"logps/chosen": -277.0523376464844,
|
|
"logps/rejected": -275.80938720703125,
|
|
"logps/weighted_chosen": -2.61376953125,
|
|
"logps/weighted_rejected": -2.787853956222534,
|
|
"loss": 0.7022,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -12.673730850219727,
|
|
"rewards/margins": 7.1806640625,
|
|
"rewards/rejected": -19.852344512939453,
|
|
"rewards/weighted_accuracies": 0.5687500238418579,
|
|
"rewards/weighted_chosen": -0.12388916313648224,
|
|
"rewards/weighted_margins": 0.19627074897289276,
|
|
"rewards/weighted_rejected": -0.3203796446323395,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.1360900287882753,
|
|
"grad_norm": 19.698871612548828,
|
|
"learning_rate": 9.962563528175875e-07,
|
|
"logits/chosen": -0.4065658450126648,
|
|
"logits/rejected": -0.4432968199253082,
|
|
"logps/chosen": -310.62890625,
|
|
"logps/rejected": -281.46405029296875,
|
|
"logps/weighted_chosen": -2.184094190597534,
|
|
"logps/weighted_rejected": -3.0492796897888184,
|
|
"loss": 0.6507,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -11.896581649780273,
|
|
"rewards/margins": 8.622265815734863,
|
|
"rewards/rejected": -20.520313262939453,
|
|
"rewards/weighted_accuracies": 0.606249988079071,
|
|
"rewards/weighted_chosen": -0.1260833740234375,
|
|
"rewards/weighted_margins": 0.25025635957717896,
|
|
"rewards/weighted_rejected": -0.3761749267578125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.14132426066474746,
|
|
"grad_norm": 16.363121032714844,
|
|
"learning_rate": 9.950574321395277e-07,
|
|
"logits/chosen": -0.42208632826805115,
|
|
"logits/rejected": -0.4458427429199219,
|
|
"logps/chosen": -305.9046936035156,
|
|
"logps/rejected": -286.06561279296875,
|
|
"logps/weighted_chosen": -2.40838623046875,
|
|
"logps/weighted_rejected": -2.7938475608825684,
|
|
"loss": 0.6573,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -16.622364044189453,
|
|
"rewards/margins": 6.233202934265137,
|
|
"rewards/rejected": -22.855077743530273,
|
|
"rewards/weighted_accuracies": 0.6156250238418579,
|
|
"rewards/weighted_chosen": -0.189697265625,
|
|
"rewards/weighted_margins": 0.27490538358688354,
|
|
"rewards/weighted_rejected": -0.4645233154296875,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.14655849254121958,
|
|
"grad_norm": 54.42692947387695,
|
|
"learning_rate": 9.936931663006413e-07,
|
|
"logits/chosen": -0.45263671875,
|
|
"logits/rejected": -0.44363707304000854,
|
|
"logps/chosen": -316.171875,
|
|
"logps/rejected": -303.3656311035156,
|
|
"logps/weighted_chosen": -2.4659423828125,
|
|
"logps/weighted_rejected": -3.0541749000549316,
|
|
"loss": 0.6068,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -13.402734756469727,
|
|
"rewards/margins": 10.619824409484863,
|
|
"rewards/rejected": -24.025390625,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.05214080959558487,
|
|
"rewards/weighted_margins": 0.40336912870407104,
|
|
"rewards/weighted_rejected": -0.455657958984375,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.1517927244176917,
|
|
"grad_norm": 39.20017623901367,
|
|
"learning_rate": 9.921640109546357e-07,
|
|
"logits/chosen": -0.42310255765914917,
|
|
"logits/rejected": -0.48920440673828125,
|
|
"logps/chosen": -283.7171936035156,
|
|
"logps/rejected": -278.1859436035156,
|
|
"logps/weighted_chosen": -2.396167039871216,
|
|
"logps/weighted_rejected": -3.5881590843200684,
|
|
"loss": 0.6649,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -16.9111328125,
|
|
"rewards/margins": 9.704492568969727,
|
|
"rewards/rejected": -26.62109375,
|
|
"rewards/weighted_accuracies": 0.6031249761581421,
|
|
"rewards/weighted_chosen": -0.130279541015625,
|
|
"rewards/weighted_margins": 0.3882461488246918,
|
|
"rewards/weighted_rejected": -0.5187179446220398,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.15702695629416383,
|
|
"grad_norm": 28.03601837158203,
|
|
"learning_rate": 9.90470476826975e-07,
|
|
"logits/chosen": -0.485189825296402,
|
|
"logits/rejected": -0.48862916231155396,
|
|
"logps/chosen": -289.09765625,
|
|
"logps/rejected": -297.625,
|
|
"logps/weighted_chosen": -2.2784485816955566,
|
|
"logps/weighted_rejected": -2.771862745285034,
|
|
"loss": 0.6608,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -19.676952362060547,
|
|
"rewards/margins": 10.679491996765137,
|
|
"rewards/rejected": -30.360937118530273,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.1669921875,
|
|
"rewards/weighted_margins": 0.28967589139938354,
|
|
"rewards/weighted_rejected": -0.4566032290458679,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.16226118817063595,
|
|
"grad_norm": 538.50927734375,
|
|
"learning_rate": 9.886131295443002e-07,
|
|
"logits/chosen": -0.654278576374054,
|
|
"logits/rejected": -0.7076683044433594,
|
|
"logps/chosen": -341.85467529296875,
|
|
"logps/rejected": -309.89764404296875,
|
|
"logps/weighted_chosen": -2.629150390625,
|
|
"logps/weighted_rejected": -2.8698973655700684,
|
|
"loss": 0.6788,
|
|
"rewards/accuracies": 0.515625,
|
|
"rewards/chosen": -60.568748474121094,
|
|
"rewards/margins": -0.7822265625,
|
|
"rewards/rejected": -59.785743713378906,
|
|
"rewards/weighted_accuracies": 0.6156250238418579,
|
|
"rewards/weighted_chosen": -0.24791869521141052,
|
|
"rewards/weighted_margins": 0.28089600801467896,
|
|
"rewards/weighted_rejected": -0.528765857219696,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.16749542004710807,
|
|
"grad_norm": 34.656883239746094,
|
|
"learning_rate": 9.865925894455166e-07,
|
|
"logits/chosen": -0.7003936767578125,
|
|
"logits/rejected": -0.719250500202179,
|
|
"logps/chosen": -326.3960876464844,
|
|
"logps/rejected": -290.3453063964844,
|
|
"logps/weighted_chosen": -2.553356885910034,
|
|
"logps/weighted_rejected": -3.10992431640625,
|
|
"loss": 0.7054,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -26.532812118530273,
|
|
"rewards/margins": 6.3564453125,
|
|
"rewards/rejected": -32.88788986206055,
|
|
"rewards/weighted_accuracies": 0.609375,
|
|
"rewards/weighted_chosen": -0.18020018935203552,
|
|
"rewards/weighted_margins": 0.3489990234375,
|
|
"rewards/weighted_rejected": -0.5293639898300171,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.17272965192358022,
|
|
"grad_norm": 61.1888542175293,
|
|
"learning_rate": 9.84409531374603e-07,
|
|
"logits/chosen": -0.6631911993026733,
|
|
"logits/rejected": -0.6448425054550171,
|
|
"logps/chosen": -324.87579345703125,
|
|
"logps/rejected": -291.71875,
|
|
"logps/weighted_chosen": -2.5611815452575684,
|
|
"logps/weighted_rejected": -3.060229539871216,
|
|
"loss": 0.6449,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -20.070018768310547,
|
|
"rewards/margins": 8.8251953125,
|
|
"rewards/rejected": -28.8876953125,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.140888974070549,
|
|
"rewards/weighted_margins": 0.3719635009765625,
|
|
"rewards/weighted_rejected": -0.5127013921737671,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.17796388380005235,
|
|
"grad_norm": 27.2315673828125,
|
|
"learning_rate": 9.820646844552219e-07,
|
|
"logits/chosen": -0.6496349573135376,
|
|
"logits/rejected": -0.7006805539131165,
|
|
"logps/chosen": -295.5882873535156,
|
|
"logps/rejected": -297.4906311035156,
|
|
"logps/weighted_chosen": -2.6988892555236816,
|
|
"logps/weighted_rejected": -2.898681640625,
|
|
"loss": 0.6788,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -19.176855087280273,
|
|
"rewards/margins": 12.649316787719727,
|
|
"rewards/rejected": -31.822460174560547,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.23918533325195312,
|
|
"rewards/weighted_margins": 0.2961669862270355,
|
|
"rewards/weighted_rejected": -0.5351837277412415,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.18319811567652447,
|
|
"grad_norm": 24.073888778686523,
|
|
"learning_rate": 9.795588318471964e-07,
|
|
"logits/chosen": -0.7137314081192017,
|
|
"logits/rejected": -0.7225399017333984,
|
|
"logps/chosen": -277.8890686035156,
|
|
"logps/rejected": -304.59063720703125,
|
|
"logps/weighted_chosen": -2.4300780296325684,
|
|
"logps/weighted_rejected": -2.771411180496216,
|
|
"loss": 0.6675,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -18.445703506469727,
|
|
"rewards/margins": 9.8720703125,
|
|
"rewards/rejected": -28.31640625,
|
|
"rewards/weighted_accuracies": 0.637499988079071,
|
|
"rewards/weighted_chosen": -0.20159301161766052,
|
|
"rewards/weighted_margins": 0.2779785096645355,
|
|
"rewards/weighted_rejected": -0.47947996854782104,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.1884323475529966,
|
|
"grad_norm": 12.750471115112305,
|
|
"learning_rate": 9.768928104849415e-07,
|
|
"logits/chosen": -0.7212737798690796,
|
|
"logits/rejected": -0.7225433588027954,
|
|
"logps/chosen": -299.53594970703125,
|
|
"logps/rejected": -275.5718688964844,
|
|
"logps/weighted_chosen": -2.667529344558716,
|
|
"logps/weighted_rejected": -2.800830125808716,
|
|
"loss": 0.6916,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -16.816015243530273,
|
|
"rewards/margins": 9.876562118530273,
|
|
"rewards/rejected": -26.690235137939453,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.15018615126609802,
|
|
"rewards/weighted_margins": 0.301962286233902,
|
|
"rewards/weighted_rejected": -0.45206451416015625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.19366657942946872,
|
|
"grad_norm": 29.441747665405273,
|
|
"learning_rate": 9.740675107979355e-07,
|
|
"logits/chosen": -0.6865798830986023,
|
|
"logits/rejected": -0.7117553949356079,
|
|
"logps/chosen": -331.06561279296875,
|
|
"logps/rejected": -300.31719970703125,
|
|
"logps/weighted_chosen": -1.9907715320587158,
|
|
"logps/weighted_rejected": -2.932177782058716,
|
|
"loss": 0.6819,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -16.022266387939453,
|
|
"rewards/margins": 9.630078315734863,
|
|
"rewards/rejected": -25.654491424560547,
|
|
"rewards/weighted_accuracies": 0.640625,
|
|
"rewards/weighted_chosen": -0.18877258896827698,
|
|
"rewards/weighted_margins": 0.2808380126953125,
|
|
"rewards/weighted_rejected": -0.4699081480503082,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.19890081130594087,
|
|
"grad_norm": 29.483524322509766,
|
|
"learning_rate": 9.71083876413323e-07,
|
|
"logits/chosen": -0.6637862920761108,
|
|
"logits/rejected": -0.669873058795929,
|
|
"logps/chosen": -322.3882751464844,
|
|
"logps/rejected": -300.85858154296875,
|
|
"logps/weighted_chosen": -2.189379930496216,
|
|
"logps/weighted_rejected": -2.9217162132263184,
|
|
"loss": 0.6846,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -20.004688262939453,
|
|
"rewards/margins": 11.246289253234863,
|
|
"rewards/rejected": -31.24609375,
|
|
"rewards/weighted_accuracies": 0.6000000238418579,
|
|
"rewards/weighted_chosen": -0.22467346489429474,
|
|
"rewards/weighted_margins": 0.2720580995082855,
|
|
"rewards/weighted_rejected": -0.4967102110385895,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.204135043182413,
|
|
"grad_norm": 20.563907623291016,
|
|
"learning_rate": 9.67942903840751e-07,
|
|
"logits/chosen": -0.7051689028739929,
|
|
"logits/rejected": -0.7537201046943665,
|
|
"logps/chosen": -324.1015625,
|
|
"logps/rejected": -310.375,
|
|
"logps/weighted_chosen": -2.397631883621216,
|
|
"logps/weighted_rejected": -2.950610399246216,
|
|
"loss": 0.6478,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -19.788671493530273,
|
|
"rewards/margins": 16.317577362060547,
|
|
"rewards/rejected": -36.111328125,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.22498169541358948,
|
|
"rewards/weighted_margins": 0.3520751893520355,
|
|
"rewards/weighted_rejected": -0.5770629644393921,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.2093692750588851,
|
|
"grad_norm": 23.1771183013916,
|
|
"learning_rate": 9.646456421395447e-07,
|
|
"logits/chosen": -0.7504974603652954,
|
|
"logits/rejected": -0.7628723382949829,
|
|
"logps/chosen": -341.2171936035156,
|
|
"logps/rejected": -343.9375,
|
|
"logps/weighted_chosen": -2.2680420875549316,
|
|
"logps/weighted_rejected": -3.0065674781799316,
|
|
"loss": 0.6746,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -22.944530487060547,
|
|
"rewards/margins": 17.315624237060547,
|
|
"rewards/rejected": -40.2587890625,
|
|
"rewards/weighted_accuracies": 0.621874988079071,
|
|
"rewards/weighted_chosen": -0.20337525010108948,
|
|
"rewards/weighted_margins": 0.216084286570549,
|
|
"rewards/weighted_rejected": -0.4196624755859375,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.21460350693535724,
|
|
"grad_norm": 14.420520782470703,
|
|
"learning_rate": 9.611931925683266e-07,
|
|
"logits/chosen": -0.7154334783554077,
|
|
"logits/rejected": -0.7491073608398438,
|
|
"logps/chosen": -331.38983154296875,
|
|
"logps/rejected": -303.3890686035156,
|
|
"logps/weighted_chosen": -2.190844774246216,
|
|
"logps/weighted_rejected": -2.7472167015075684,
|
|
"loss": 0.6135,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -25.111621856689453,
|
|
"rewards/margins": 14.543554306030273,
|
|
"rewards/rejected": -39.658592224121094,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.21762695908546448,
|
|
"rewards/weighted_margins": 0.36674195528030396,
|
|
"rewards/weighted_rejected": -0.584503173828125,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.21983773881182936,
|
|
"grad_norm": 27.429603576660156,
|
|
"learning_rate": 9.575867082172085e-07,
|
|
"logits/chosen": -0.7379547357559204,
|
|
"logits/rejected": -0.7826202511787415,
|
|
"logps/chosen": -337.46405029296875,
|
|
"logps/rejected": -320.23907470703125,
|
|
"logps/weighted_chosen": -2.6387085914611816,
|
|
"logps/weighted_rejected": -2.712329149246216,
|
|
"loss": 0.6716,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -32.554298400878906,
|
|
"rewards/margins": 17.513866424560547,
|
|
"rewards/rejected": -50.060157775878906,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.35613709688186646,
|
|
"rewards/weighted_margins": 0.38392335176467896,
|
|
"rewards/weighted_rejected": -0.7401062250137329,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.22507197068830148,
|
|
"grad_norm": 18.97144889831543,
|
|
"learning_rate": 9.538273936226673e-07,
|
|
"logits/chosen": -0.778491199016571,
|
|
"logits/rejected": -0.811004638671875,
|
|
"logps/chosen": -292.83984375,
|
|
"logps/rejected": -304.05352783203125,
|
|
"logps/weighted_chosen": -2.796630859375,
|
|
"logps/weighted_rejected": -3.2444825172424316,
|
|
"loss": 0.6544,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -27.327733993530273,
|
|
"rewards/margins": 12.26318359375,
|
|
"rewards/rejected": -39.58984375,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.21069030463695526,
|
|
"rewards/weighted_margins": 0.3678832948207855,
|
|
"rewards/weighted_rejected": -0.57806396484375,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.23030620256477363,
|
|
"grad_norm": 25.43462371826172,
|
|
"learning_rate": 9.499165043652391e-07,
|
|
"logits/chosen": -0.7674010992050171,
|
|
"logits/rejected": -0.7686828374862671,
|
|
"logps/chosen": -319.55859375,
|
|
"logps/rejected": -309.03436279296875,
|
|
"logps/weighted_chosen": -2.82305908203125,
|
|
"logps/weighted_rejected": -2.993237257003784,
|
|
"loss": 0.631,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -28.386133193969727,
|
|
"rewards/margins": 13.9384765625,
|
|
"rewards/rejected": -42.326072692871094,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.26903897523880005,
|
|
"rewards/weighted_margins": 0.353515625,
|
|
"rewards/weighted_rejected": -0.622546374797821,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.23554043444124576,
|
|
"grad_norm": 65.78443908691406,
|
|
"learning_rate": 9.458553466501665e-07,
|
|
"logits/chosen": -0.8066772222518921,
|
|
"logits/rejected": -0.8363037109375,
|
|
"logps/chosen": -314.7945251464844,
|
|
"logps/rejected": -287.65313720703125,
|
|
"logps/weighted_chosen": -2.8233399391174316,
|
|
"logps/weighted_rejected": -3.013622999191284,
|
|
"loss": 0.6831,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -28.642578125,
|
|
"rewards/margins": 15.162694931030273,
|
|
"rewards/rejected": -43.80976486206055,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.3802246153354645,
|
|
"rewards/weighted_margins": 0.3342132568359375,
|
|
"rewards/weighted_rejected": -0.7139984369277954,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.24077466631771788,
|
|
"grad_norm": 13.290085792541504,
|
|
"learning_rate": 9.416452768711366e-07,
|
|
"logits/chosen": -0.7957550287246704,
|
|
"logits/rejected": -0.8287414312362671,
|
|
"logps/chosen": -323.0093688964844,
|
|
"logps/rejected": -306.98907470703125,
|
|
"logps/weighted_chosen": -2.544872999191284,
|
|
"logps/weighted_rejected": -3.100903272628784,
|
|
"loss": 0.6759,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -30.906835556030273,
|
|
"rewards/margins": 16.355859756469727,
|
|
"rewards/rejected": -47.24980545043945,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.343759149312973,
|
|
"rewards/weighted_margins": 0.40337830781936646,
|
|
"rewards/weighted_rejected": -0.747100830078125,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.24600889819419,
|
|
"grad_norm": 21.766939163208008,
|
|
"learning_rate": 9.372877011572557e-07,
|
|
"logits/chosen": -0.7200164794921875,
|
|
"logits/rejected": -0.742279052734375,
|
|
"logps/chosen": -342.75079345703125,
|
|
"logps/rejected": -318.60626220703125,
|
|
"logps/weighted_chosen": -2.5311522483825684,
|
|
"logps/weighted_rejected": -2.990124464035034,
|
|
"loss": 0.63,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -32.077247619628906,
|
|
"rewards/margins": 11.883398056030273,
|
|
"rewards/rejected": -43.959373474121094,
|
|
"rewards/weighted_accuracies": 0.6656249761581421,
|
|
"rewards/weighted_chosen": -0.30525511503219604,
|
|
"rewards/weighted_margins": 0.45678406953811646,
|
|
"rewards/weighted_rejected": -0.7624969482421875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.2512431300706621,
|
|
"grad_norm": 28.76239776611328,
|
|
"learning_rate": 9.327840749034141e-07,
|
|
"logits/chosen": -0.7930053472518921,
|
|
"logits/rejected": -0.8311401605606079,
|
|
"logps/chosen": -316.79998779296875,
|
|
"logps/rejected": -326.5062561035156,
|
|
"logps/weighted_chosen": -2.4120116233825684,
|
|
"logps/weighted_rejected": -3.591870069503784,
|
|
"loss": 0.6639,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -29.640039443969727,
|
|
"rewards/margins": 20.350976943969727,
|
|
"rewards/rejected": -49.9853515625,
|
|
"rewards/weighted_accuracies": 0.6812499761581421,
|
|
"rewards/weighted_chosen": -0.27521055936813354,
|
|
"rewards/weighted_margins": 0.4522705078125,
|
|
"rewards/weighted_rejected": -0.7274719476699829,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.2564773619471343,
|
|
"grad_norm": 36.629127502441406,
|
|
"learning_rate": 9.281359022841965e-07,
|
|
"logits/chosen": -0.72747802734375,
|
|
"logits/rejected": -0.7426910400390625,
|
|
"logps/chosen": -308.5406188964844,
|
|
"logps/rejected": -300.71484375,
|
|
"logps/weighted_chosen": -2.6044554710388184,
|
|
"logps/weighted_rejected": -3.862866163253784,
|
|
"loss": 0.6178,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -33.54804611206055,
|
|
"rewards/margins": 22.3515625,
|
|
"rewards/rejected": -55.88496017456055,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.38198548555374146,
|
|
"rewards/weighted_margins": 0.525561511516571,
|
|
"rewards/weighted_rejected": -0.9073349237442017,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"grad_norm": 20.296154022216797,
|
|
"learning_rate": 9.233447357514989e-07,
|
|
"logits/chosen": -0.7092193365097046,
|
|
"logits/rejected": -0.751629650592804,
|
|
"logps/chosen": -337.10467529296875,
|
|
"logps/rejected": -328.71875,
|
|
"logps/weighted_chosen": -3.054370164871216,
|
|
"logps/weighted_rejected": -3.5334715843200684,
|
|
"loss": 0.6534,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -39.15234375,
|
|
"rewards/margins": 19.770116806030273,
|
|
"rewards/rejected": -58.90898513793945,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.470510870218277,
|
|
"rewards/weighted_margins": 0.566607654094696,
|
|
"rewards/weighted_rejected": -1.0376465320587158,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"eval_logits/chosen": -0.8052441477775574,
|
|
"eval_logits/rejected": -0.8225547075271606,
|
|
"eval_logps/chosen": -333.44000244140625,
|
|
"eval_logps/rejected": -331.98199462890625,
|
|
"eval_logps/weighted_chosen": -2.756896734237671,
|
|
"eval_logps/weighted_rejected": -3.441680908203125,
|
|
"eval_loss": 0.6561886668205261,
|
|
"eval_rewards/accuracies": 0.6370000243186951,
|
|
"eval_rewards/chosen": -44.67877960205078,
|
|
"eval_rewards/margins": 19.602703094482422,
|
|
"eval_rewards/rejected": -64.27362823486328,
|
|
"eval_rewards/weighted_accuracies": 0.6445000171661377,
|
|
"eval_rewards/weighted_chosen": -0.485819548368454,
|
|
"eval_rewards/weighted_margins": 0.4620407819747925,
|
|
"eval_rewards/weighted_rejected": -0.9478604793548584,
|
|
"eval_runtime": 1263.3333,
|
|
"eval_samples_per_second": 1.583,
|
|
"eval_steps_per_second": 0.396,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.2669458257000785,
|
|
"grad_norm": 44.70832824707031,
|
|
"learning_rate": 9.184121755160232e-07,
|
|
"logits/chosen": -0.7849181890487671,
|
|
"logits/rejected": -0.8171790838241577,
|
|
"logps/chosen": -344.84454345703125,
|
|
"logps/rejected": -356.3671875,
|
|
"logps/weighted_chosen": -2.9210448265075684,
|
|
"logps/weighted_rejected": -3.3969483375549316,
|
|
"loss": 0.6735,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -43.604881286621094,
|
|
"rewards/margins": 24.157032012939453,
|
|
"rewards/rejected": -67.75703430175781,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.43794554471969604,
|
|
"rewards/weighted_margins": 0.407052606344223,
|
|
"rewards/weighted_rejected": -0.8448547124862671,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.2721800575765506,
|
|
"grad_norm": 34.907981872558594,
|
|
"learning_rate": 9.133398690128193e-07,
|
|
"logits/chosen": -0.8243468999862671,
|
|
"logits/rejected": -0.852618396282196,
|
|
"logps/chosen": -370.2640686035156,
|
|
"logps/rejected": -357.4296875,
|
|
"logps/weighted_chosen": -2.7059326171875,
|
|
"logps/weighted_rejected": -3.559033155441284,
|
|
"loss": 0.6233,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -48.807029724121094,
|
|
"rewards/margins": 28.050586700439453,
|
|
"rewards/rejected": -76.85234069824219,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.3096374571323395,
|
|
"rewards/weighted_margins": 0.5285431146621704,
|
|
"rewards/weighted_rejected": -0.838287353515625,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.27741428945302277,
|
|
"grad_norm": 17.412511825561523,
|
|
"learning_rate": 9.081295103510554e-07,
|
|
"logits/chosen": -0.7943557500839233,
|
|
"logits/rejected": -0.8541763424873352,
|
|
"logps/chosen": -339.65313720703125,
|
|
"logps/rejected": -351.77813720703125,
|
|
"logps/weighted_chosen": -2.331298828125,
|
|
"logps/weighted_rejected": -3.5838379859924316,
|
|
"loss": 0.5587,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -55.494140625,
|
|
"rewards/margins": 26.642187118530273,
|
|
"rewards/rejected": -82.14140319824219,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.33439940214157104,
|
|
"rewards/weighted_margins": 0.7252563238143921,
|
|
"rewards/weighted_rejected": -1.0597717761993408,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.2826485213294949,
|
|
"grad_norm": 30.677711486816406,
|
|
"learning_rate": 9.027828397481989e-07,
|
|
"logits/chosen": -0.7925201654434204,
|
|
"logits/rejected": -0.8262939453125,
|
|
"logps/chosen": -318.59063720703125,
|
|
"logps/rejected": -337.55157470703125,
|
|
"logps/weighted_chosen": -3.059436082839966,
|
|
"logps/weighted_rejected": -3.750012159347534,
|
|
"loss": 0.6464,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -57.99492263793945,
|
|
"rewards/margins": 24.116796493530273,
|
|
"rewards/rejected": -82.107421875,
|
|
"rewards/weighted_accuracies": 0.637499988079071,
|
|
"rewards/weighted_chosen": -0.507769763469696,
|
|
"rewards/weighted_margins": 0.4883270263671875,
|
|
"rewards/weighted_rejected": -0.995800793170929,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.287882753205967,
|
|
"grad_norm": 34.808658599853516,
|
|
"learning_rate": 8.973016429487988e-07,
|
|
"logits/chosen": -0.8280746340751648,
|
|
"logits/rejected": -0.8393570184707642,
|
|
"logps/chosen": -340.02032470703125,
|
|
"logps/rejected": -340.90625,
|
|
"logps/weighted_chosen": -3.016833543777466,
|
|
"logps/weighted_rejected": -3.3585205078125,
|
|
"loss": 0.6373,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -58.115234375,
|
|
"rewards/margins": 28.575389862060547,
|
|
"rewards/rejected": -86.7035140991211,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.5330657958984375,
|
|
"rewards/weighted_margins": 0.48836976289749146,
|
|
"rewards/weighted_rejected": -1.0212554931640625,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.29311698508243916,
|
|
"grad_norm": 21.559553146362305,
|
|
"learning_rate": 8.916877506280601e-07,
|
|
"logits/chosen": -0.8576828241348267,
|
|
"logits/rejected": -0.850115954875946,
|
|
"logps/chosen": -343.0625,
|
|
"logps/rejected": -340.73126220703125,
|
|
"logps/weighted_chosen": -3.004504442214966,
|
|
"logps/weighted_rejected": -3.3214111328125,
|
|
"loss": 0.6493,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -62.214454650878906,
|
|
"rewards/margins": 24.381053924560547,
|
|
"rewards/rejected": -86.59492492675781,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.5099731683731079,
|
|
"rewards/weighted_margins": 0.535810112953186,
|
|
"rewards/weighted_rejected": -1.0458984375,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.29835121695891126,
|
|
"grad_norm": 16.77034568786621,
|
|
"learning_rate": 8.85943037780415e-07,
|
|
"logits/chosen": -0.901629626750946,
|
|
"logits/rejected": -0.9031143188476562,
|
|
"logps/chosen": -347.7562561035156,
|
|
"logps/rejected": -320.90936279296875,
|
|
"logps/weighted_chosen": -2.9189209938049316,
|
|
"logps/weighted_rejected": -3.346874952316284,
|
|
"loss": 0.6796,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -61.52226638793945,
|
|
"rewards/margins": 17.764842987060547,
|
|
"rewards/rejected": -79.3128890991211,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.5591338872909546,
|
|
"rewards/weighted_margins": 0.44241029024124146,
|
|
"rewards/weighted_rejected": -1.00177001953125,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.3035854488353834,
|
|
"grad_norm": 23.179088592529297,
|
|
"learning_rate": 8.800694230932884e-07,
|
|
"logits/chosen": -0.808392345905304,
|
|
"logits/rejected": -0.8254486322402954,
|
|
"logps/chosen": -345.52032470703125,
|
|
"logps/rejected": -338.59844970703125,
|
|
"logps/weighted_chosen": -2.4705810546875,
|
|
"logps/weighted_rejected": -3.031982421875,
|
|
"loss": 0.6672,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -56.009376525878906,
|
|
"rewards/margins": 17.356250762939453,
|
|
"rewards/rejected": -73.3921890258789,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.3469276428222656,
|
|
"rewards/weighted_margins": 0.38211363554000854,
|
|
"rewards/weighted_rejected": -0.7289062738418579,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.30881968071185556,
|
|
"grad_norm": 16.077539443969727,
|
|
"learning_rate": 8.740688683062723e-07,
|
|
"logits/chosen": -0.8602691888809204,
|
|
"logits/rejected": -0.874432384967804,
|
|
"logps/chosen": -382.03436279296875,
|
|
"logps/rejected": -349.27655029296875,
|
|
"logps/weighted_chosen": -2.4807372093200684,
|
|
"logps/weighted_rejected": -3.101879835128784,
|
|
"loss": 0.6615,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -52.95586013793945,
|
|
"rewards/margins": 20.580469131469727,
|
|
"rewards/rejected": -73.5445327758789,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.31566160917282104,
|
|
"rewards/weighted_margins": 0.39473265409469604,
|
|
"rewards/weighted_rejected": -0.7103912234306335,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.31405391258832765,
|
|
"grad_norm": 17.0419864654541,
|
|
"learning_rate": 8.679433775559215e-07,
|
|
"logits/chosen": -0.8191520571708679,
|
|
"logits/rejected": -0.8663116693496704,
|
|
"logps/chosen": -379.78125,
|
|
"logps/rejected": -361.57501220703125,
|
|
"logps/weighted_chosen": -2.305920362472534,
|
|
"logps/weighted_rejected": -3.3094482421875,
|
|
"loss": 0.6241,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -55.09687423706055,
|
|
"rewards/margins": 20.994531631469727,
|
|
"rewards/rejected": -76.0738296508789,
|
|
"rewards/weighted_accuracies": 0.653124988079071,
|
|
"rewards/weighted_chosen": -0.31669920682907104,
|
|
"rewards/weighted_margins": 0.4563964903354645,
|
|
"rewards/weighted_rejected": -0.7728790044784546,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.3192881444647998,
|
|
"grad_norm": 30.722089767456055,
|
|
"learning_rate": 8.616949967063871e-07,
|
|
"logits/chosen": -0.7851959466934204,
|
|
"logits/rejected": -0.82568359375,
|
|
"logps/chosen": -323.5859375,
|
|
"logps/rejected": -338.609375,
|
|
"logps/weighted_chosen": -2.7684326171875,
|
|
"logps/weighted_rejected": -3.191943407058716,
|
|
"loss": 0.6918,
|
|
"rewards/accuracies": 0.684374988079071,
|
|
"rewards/chosen": -55.26250076293945,
|
|
"rewards/margins": 22.568164825439453,
|
|
"rewards/rejected": -77.8238296508789,
|
|
"rewards/weighted_accuracies": 0.625,
|
|
"rewards/weighted_chosen": -0.423666387796402,
|
|
"rewards/weighted_margins": 0.310333251953125,
|
|
"rewards/weighted_rejected": -0.733630359172821,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.3245223763412719,
|
|
"grad_norm": 14.894518852233887,
|
|
"learning_rate": 8.553258126661154e-07,
|
|
"logits/chosen": -0.831768810749054,
|
|
"logits/rejected": -0.846484363079071,
|
|
"logps/chosen": -338.09686279296875,
|
|
"logps/rejected": -336.015625,
|
|
"logps/weighted_chosen": -2.84112548828125,
|
|
"logps/weighted_rejected": -3.4341063499450684,
|
|
"loss": 0.708,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -58.20390701293945,
|
|
"rewards/margins": 21.783594131469727,
|
|
"rewards/rejected": -79.9749984741211,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.4535583555698395,
|
|
"rewards/weighted_margins": 0.33486634492874146,
|
|
"rewards/weighted_rejected": -0.788104236125946,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.32975660821774405,
|
|
"grad_norm": 23.813823699951172,
|
|
"learning_rate": 8.488379526908368e-07,
|
|
"logits/chosen": -0.826812744140625,
|
|
"logits/rejected": -0.837506115436554,
|
|
"logps/chosen": -352.4593811035156,
|
|
"logps/rejected": -357.29376220703125,
|
|
"logps/weighted_chosen": -2.6490235328674316,
|
|
"logps/weighted_rejected": -3.211181640625,
|
|
"loss": 0.6454,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -61.892189025878906,
|
|
"rewards/margins": 26.424219131469727,
|
|
"rewards/rejected": -88.33320617675781,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.3862060606479645,
|
|
"rewards/weighted_margins": 0.415771484375,
|
|
"rewards/weighted_rejected": -0.8020385503768921,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.33499084009421615,
|
|
"grad_norm": 19.41891098022461,
|
|
"learning_rate": 8.422335836730802e-07,
|
|
"logits/chosen": -0.7994629144668579,
|
|
"logits/rejected": -0.7995041012763977,
|
|
"logps/chosen": -333.2593688964844,
|
|
"logps/rejected": -366.6499938964844,
|
|
"logps/weighted_chosen": -2.6496825218200684,
|
|
"logps/weighted_rejected": -3.1250548362731934,
|
|
"loss": 0.6734,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -59.58320236206055,
|
|
"rewards/margins": 29.476757049560547,
|
|
"rewards/rejected": -89.0625,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.47590941190719604,
|
|
"rewards/weighted_margins": 0.394134521484375,
|
|
"rewards/weighted_rejected": -0.8701080083847046,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.3402250719706883,
|
|
"grad_norm": 16.819276809692383,
|
|
"learning_rate": 8.355149114184485e-07,
|
|
"logits/chosen": -0.846386730670929,
|
|
"logits/rejected": -0.8338836431503296,
|
|
"logps/chosen": -370.0062561035156,
|
|
"logps/rejected": -376.0843811035156,
|
|
"logps/weighted_chosen": -2.787951707839966,
|
|
"logps/weighted_rejected": -3.106738328933716,
|
|
"loss": 0.6483,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -62.594337463378906,
|
|
"rewards/margins": 31.204687118530273,
|
|
"rewards/rejected": -93.80000305175781,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.5348541140556335,
|
|
"rewards/weighted_margins": 0.446258544921875,
|
|
"rewards/weighted_rejected": -0.980926513671875,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.34545930384716045,
|
|
"grad_norm": 18.66504669189453,
|
|
"learning_rate": 8.286841799088963e-07,
|
|
"logits/chosen": -0.8683761358261108,
|
|
"logits/rejected": -0.860211193561554,
|
|
"logps/chosen": -344.94061279296875,
|
|
"logps/rejected": -343.61407470703125,
|
|
"logps/weighted_chosen": -2.3468871116638184,
|
|
"logps/weighted_rejected": -2.966168165206909,
|
|
"loss": 0.6577,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -60.857032775878906,
|
|
"rewards/margins": 19.649999618530273,
|
|
"rewards/rejected": -80.50117492675781,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.4206695556640625,
|
|
"rewards/weighted_margins": 0.41551512479782104,
|
|
"rewards/weighted_rejected": -0.8359512090682983,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.35069353572363254,
|
|
"grad_norm": 26.941055297851562,
|
|
"learning_rate": 8.217436705532599e-07,
|
|
"logits/chosen": -0.8248077630996704,
|
|
"logits/rejected": -0.8512848019599915,
|
|
"logps/chosen": -369.5484313964844,
|
|
"logps/rejected": -348.30157470703125,
|
|
"logps/weighted_chosen": -2.3807740211486816,
|
|
"logps/weighted_rejected": -3.006176710128784,
|
|
"loss": 0.6373,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -66.71875,
|
|
"rewards/margins": 17.846094131469727,
|
|
"rewards/rejected": -84.57929992675781,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.3303161561489105,
|
|
"rewards/weighted_margins": 0.45032960176467896,
|
|
"rewards/weighted_rejected": -0.7810913324356079,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.3559277676001047,
|
|
"grad_norm": 1516.0828857421875,
|
|
"learning_rate": 8.14695701425284e-07,
|
|
"logits/chosen": -0.8020523190498352,
|
|
"logits/rejected": -0.845899224281311,
|
|
"logps/chosen": -371.13751220703125,
|
|
"logps/rejected": -352.38592529296875,
|
|
"logps/weighted_chosen": -2.49072265625,
|
|
"logps/weighted_rejected": -3.2708497047424316,
|
|
"loss": 0.5885,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -65.716796875,
|
|
"rewards/margins": 24.369531631469727,
|
|
"rewards/rejected": -90.1097640991211,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.330526739358902,
|
|
"rewards/weighted_margins": 0.5371948480606079,
|
|
"rewards/weighted_rejected": -0.867462158203125,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.3611619994765768,
|
|
"grad_norm": 295.9232482910156,
|
|
"learning_rate": 8.075426264894046e-07,
|
|
"logits/chosen": -0.7686309814453125,
|
|
"logits/rejected": -0.805737316608429,
|
|
"logps/chosen": -370.75,
|
|
"logps/rejected": -373.64373779296875,
|
|
"logps/weighted_chosen": -2.509265184402466,
|
|
"logps/weighted_rejected": -3.719970703125,
|
|
"loss": 0.5532,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -65.3414077758789,
|
|
"rewards/margins": 30.711523056030273,
|
|
"rewards/rejected": -96.052734375,
|
|
"rewards/weighted_accuracies": 0.7406250238418579,
|
|
"rewards/weighted_chosen": -0.3384948670864105,
|
|
"rewards/weighted_margins": 0.6478027105331421,
|
|
"rewards/weighted_rejected": -0.9860213994979858,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.36639623135304894,
|
|
"grad_norm": 80.06324768066406,
|
|
"learning_rate": 8.002868348145435e-07,
|
|
"logits/chosen": -0.7615035772323608,
|
|
"logits/rejected": -0.758954644203186,
|
|
"logps/chosen": -364.390625,
|
|
"logps/rejected": -353.75,
|
|
"logps/weighted_chosen": -2.562756299972534,
|
|
"logps/weighted_rejected": -2.821521043777466,
|
|
"loss": 0.6219,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -64.1957015991211,
|
|
"rewards/margins": 21.513866424560547,
|
|
"rewards/rejected": -85.70429992675781,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.38551026582717896,
|
|
"rewards/weighted_margins": 0.45988160371780396,
|
|
"rewards/weighted_rejected": -0.8446716070175171,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.3716304632295211,
|
|
"grad_norm": 1476.4896240234375,
|
|
"learning_rate": 7.92930749776179e-07,
|
|
"logits/chosen": -0.75201416015625,
|
|
"logits/rejected": -0.7803069949150085,
|
|
"logps/chosen": -337.1851501464844,
|
|
"logps/rejected": -346.5375061035156,
|
|
"logps/weighted_chosen": -2.752087354660034,
|
|
"logps/weighted_rejected": -3.417065382003784,
|
|
"loss": 0.6452,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -65.52070617675781,
|
|
"rewards/margins": 20.681640625,
|
|
"rewards/rejected": -86.203125,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.2934509217739105,
|
|
"rewards/weighted_margins": 0.537158191204071,
|
|
"rewards/weighted_rejected": -0.8301132321357727,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.3768646951059932,
|
|
"grad_norm": 37.869117736816406,
|
|
"learning_rate": 7.854768282469582e-07,
|
|
"logits/chosen": -0.814867377281189,
|
|
"logits/rejected": -0.8494598269462585,
|
|
"logps/chosen": -332.6953125,
|
|
"logps/rejected": -362.8187561035156,
|
|
"logps/weighted_chosen": -2.5288939476013184,
|
|
"logps/weighted_rejected": -3.1172118186950684,
|
|
"loss": 0.6496,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -55.247657775878906,
|
|
"rewards/margins": 29.8330078125,
|
|
"rewards/rejected": -85.080078125,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.355978399515152,
|
|
"rewards/weighted_margins": 0.4522338807582855,
|
|
"rewards/weighted_rejected": -0.8084503412246704,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.38209892698246534,
|
|
"grad_norm": 48.1231575012207,
|
|
"learning_rate": 7.779275597761215e-07,
|
|
"logits/chosen": -0.7673202753067017,
|
|
"logits/rejected": -0.81195068359375,
|
|
"logps/chosen": -330.94451904296875,
|
|
"logps/rejected": -355.31561279296875,
|
|
"logps/weighted_chosen": -2.6014404296875,
|
|
"logps/weighted_rejected": -3.1932616233825684,
|
|
"loss": 0.5952,
|
|
"rewards/accuracies": 0.690625011920929,
|
|
"rewards/chosen": -54.890235900878906,
|
|
"rewards/margins": 31.880468368530273,
|
|
"rewards/rejected": -86.75312805175781,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.2228240966796875,
|
|
"rewards/weighted_margins": 0.6149749755859375,
|
|
"rewards/weighted_rejected": -0.8373657464981079,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.38733315885893743,
|
|
"grad_norm": 296.0299377441406,
|
|
"learning_rate": 7.702854657580126e-07,
|
|
"logits/chosen": -0.8295089602470398,
|
|
"logits/rejected": -0.843798816204071,
|
|
"logps/chosen": -352.19842529296875,
|
|
"logps/rejected": -333.7593688964844,
|
|
"logps/weighted_chosen": -2.45697021484375,
|
|
"logps/weighted_rejected": -3.375244140625,
|
|
"loss": 0.6318,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -57.54804611206055,
|
|
"rewards/margins": 21.8720703125,
|
|
"rewards/rejected": -79.4625015258789,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.19748535752296448,
|
|
"rewards/weighted_margins": 0.5647827386856079,
|
|
"rewards/weighted_rejected": -0.762377917766571,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.3925673907354096,
|
|
"grad_norm": 222.2068328857422,
|
|
"learning_rate": 7.625530985899547e-07,
|
|
"logits/chosen": -0.8145691156387329,
|
|
"logits/rejected": -0.8263305425643921,
|
|
"logps/chosen": -328.7578125,
|
|
"logps/rejected": -331.390625,
|
|
"logps/weighted_chosen": -2.581188917160034,
|
|
"logps/weighted_rejected": -3.505688428878784,
|
|
"loss": 0.6377,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -61.987892150878906,
|
|
"rewards/margins": 24.421483993530273,
|
|
"rewards/rejected": -86.39921569824219,
|
|
"rewards/weighted_accuracies": 0.6656249761581421,
|
|
"rewards/weighted_chosen": -0.493093878030777,
|
|
"rewards/weighted_margins": 0.4742370545864105,
|
|
"rewards/weighted_rejected": -0.9672302007675171,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.39780162261188173,
|
|
"grad_norm": 34.57517623901367,
|
|
"learning_rate": 7.547330408197694e-07,
|
|
"logits/chosen": -0.8249969482421875,
|
|
"logits/rejected": -0.8720428347587585,
|
|
"logps/chosen": -363.44219970703125,
|
|
"logps/rejected": -345.2250061035156,
|
|
"logps/weighted_chosen": -2.4618163108825684,
|
|
"logps/weighted_rejected": -3.28759765625,
|
|
"loss": 0.6383,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -67.7855453491211,
|
|
"rewards/margins": 21.181835174560547,
|
|
"rewards/rejected": -88.9535140991211,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.3757568299770355,
|
|
"rewards/weighted_margins": 0.4518585205078125,
|
|
"rewards/weighted_rejected": -0.8270477056503296,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.40303585448835383,
|
|
"grad_norm": 26.55402946472168,
|
|
"learning_rate": 7.468279042832271e-07,
|
|
"logits/chosen": -0.8424628973007202,
|
|
"logits/rejected": -0.8771301507949829,
|
|
"logps/chosen": -347.6484375,
|
|
"logps/rejected": -388.42266845703125,
|
|
"logps/weighted_chosen": -2.660815477371216,
|
|
"logps/weighted_rejected": -3.1720213890075684,
|
|
"loss": 0.6743,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -67.98554992675781,
|
|
"rewards/margins": 30.81640625,
|
|
"rewards/rejected": -98.75859069824219,
|
|
"rewards/weighted_accuracies": 0.6312500238418579,
|
|
"rewards/weighted_chosen": -0.5194793939590454,
|
|
"rewards/weighted_margins": 0.3566345274448395,
|
|
"rewards/weighted_rejected": -0.8764098882675171,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.408270086364826,
|
|
"grad_norm": 14.42599868774414,
|
|
"learning_rate": 7.388403292317154e-07,
|
|
"logits/chosen": -0.7979522943496704,
|
|
"logits/rejected": -0.8573578000068665,
|
|
"logps/chosen": -367.4046936035156,
|
|
"logps/rejected": -358.2515563964844,
|
|
"logps/weighted_chosen": -2.490283250808716,
|
|
"logps/weighted_rejected": -3.1418213844299316,
|
|
"loss": 0.6398,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -66.2933578491211,
|
|
"rewards/margins": 26.066015243530273,
|
|
"rewards/rejected": -92.384765625,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.461639404296875,
|
|
"rewards/weighted_margins": 0.452981561422348,
|
|
"rewards/weighted_rejected": -0.914324939250946,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.4135043182412981,
|
|
"grad_norm": 21.45990753173828,
|
|
"learning_rate": 7.307729834504154e-07,
|
|
"logits/chosen": -0.8032287359237671,
|
|
"logits/rejected": -0.8670104742050171,
|
|
"logps/chosen": -351.39532470703125,
|
|
"logps/rejected": -360.1734313964844,
|
|
"logps/weighted_chosen": -2.3328614234924316,
|
|
"logps/weighted_rejected": -3.198779344558716,
|
|
"loss": 0.6356,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -69.6167984008789,
|
|
"rewards/margins": 24.462499618530273,
|
|
"rewards/rejected": -94.07890319824219,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.3527267575263977,
|
|
"rewards/weighted_margins": 0.45725250244140625,
|
|
"rewards/weighted_rejected": -0.8100005984306335,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.4187385501177702,
|
|
"grad_norm": 20.13976287841797,
|
|
"learning_rate": 7.226285613672847e-07,
|
|
"logits/chosen": -0.741473376750946,
|
|
"logits/rejected": -0.7820758819580078,
|
|
"logps/chosen": -342.1156311035156,
|
|
"logps/rejected": -382.6937561035156,
|
|
"logps/weighted_chosen": -2.3370361328125,
|
|
"logps/weighted_rejected": -3.3424315452575684,
|
|
"loss": 0.6236,
|
|
"rewards/accuracies": 0.703125,
|
|
"rewards/chosen": -64.8167953491211,
|
|
"rewards/margins": 39.174217224121094,
|
|
"rewards/rejected": -103.96992492675781,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.34785765409469604,
|
|
"rewards/weighted_margins": 0.5167236328125,
|
|
"rewards/weighted_rejected": -0.864398181438446,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.4239727819942423,
|
|
"grad_norm": 21.571788787841797,
|
|
"learning_rate": 7.144097831531398e-07,
|
|
"logits/chosen": -0.6900985836982727,
|
|
"logits/rejected": -0.7189788818359375,
|
|
"logps/chosen": -344.78436279296875,
|
|
"logps/rejected": -364.53436279296875,
|
|
"logps/weighted_chosen": -2.346606492996216,
|
|
"logps/weighted_rejected": -3.139209032058716,
|
|
"loss": 0.6171,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -74.72969055175781,
|
|
"rewards/margins": 25.596094131469727,
|
|
"rewards/rejected": -100.31640625,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.332855224609375,
|
|
"rewards/weighted_margins": 0.513507068157196,
|
|
"rewards/weighted_rejected": -0.8466736078262329,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.42920701387071447,
|
|
"grad_norm": 23.761091232299805,
|
|
"learning_rate": 7.061193938131396e-07,
|
|
"logits/chosen": -0.620227038860321,
|
|
"logits/rejected": -0.6747413873672485,
|
|
"logps/chosen": -377.5609436035156,
|
|
"logps/rejected": -363.4296875,
|
|
"logps/weighted_chosen": -2.772265672683716,
|
|
"logps/weighted_rejected": -3.069580078125,
|
|
"loss": 0.6365,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -77.7945327758789,
|
|
"rewards/margins": 20.221874237060547,
|
|
"rewards/rejected": -97.98515319824219,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.3748535215854645,
|
|
"rewards/weighted_margins": 0.473724365234375,
|
|
"rewards/weighted_rejected": -0.8482757806777954,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.4344412457471866,
|
|
"grad_norm": 21.571779251098633,
|
|
"learning_rate": 6.977601622699789e-07,
|
|
"logits/chosen": -0.689013659954071,
|
|
"logits/rejected": -0.7498534917831421,
|
|
"logps/chosen": -354.41876220703125,
|
|
"logps/rejected": -392.3500061035156,
|
|
"logps/weighted_chosen": -2.6583251953125,
|
|
"logps/weighted_rejected": -3.377002000808716,
|
|
"loss": 0.5618,
|
|
"rewards/accuracies": 0.7093750238418579,
|
|
"rewards/chosen": -69.55390930175781,
|
|
"rewards/margins": 43.986717224121094,
|
|
"rewards/rejected": -113.5484390258789,
|
|
"rewards/weighted_accuracies": 0.7124999761581421,
|
|
"rewards/weighted_chosen": -0.28594970703125,
|
|
"rewards/weighted_margins": 0.6694701910018921,
|
|
"rewards/weighted_rejected": -0.955474853515625,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.4396754776236587,
|
|
"grad_norm": 21.845787048339844,
|
|
"learning_rate": 6.893348804390882e-07,
|
|
"logits/chosen": -0.7911956906318665,
|
|
"logits/rejected": -0.8087249994277954,
|
|
"logps/chosen": -377.0531311035156,
|
|
"logps/rejected": -377.48126220703125,
|
|
"logps/weighted_chosen": -2.844287157058716,
|
|
"logps/weighted_rejected": -3.24560546875,
|
|
"loss": 0.5927,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -81.9203109741211,
|
|
"rewards/margins": 35.79375076293945,
|
|
"rewards/rejected": -117.70625305175781,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -0.3613952696323395,
|
|
"rewards/weighted_margins": 0.5852203369140625,
|
|
"rewards/weighted_rejected": -0.94671630859375,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.44490970950013087,
|
|
"grad_norm": 13.673724174499512,
|
|
"learning_rate": 6.808463622961578e-07,
|
|
"logits/chosen": -0.765423595905304,
|
|
"logits/rejected": -0.8230966329574585,
|
|
"logps/chosen": -385.33905029296875,
|
|
"logps/rejected": -413.21563720703125,
|
|
"logps/weighted_chosen": -2.7145752906799316,
|
|
"logps/weighted_rejected": -3.412890672683716,
|
|
"loss": 0.5718,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -92.6429672241211,
|
|
"rewards/margins": 38.67695236206055,
|
|
"rewards/rejected": -131.3136749267578,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.43825072050094604,
|
|
"rewards/weighted_margins": 0.649249255657196,
|
|
"rewards/weighted_rejected": -1.0877685546875,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.45014394137660296,
|
|
"grad_norm": 17.156784057617188,
|
|
"learning_rate": 6.722974429372925e-07,
|
|
"logits/chosen": -0.733477771282196,
|
|
"logits/rejected": -0.7933975458145142,
|
|
"logps/chosen": -418.08282470703125,
|
|
"logps/rejected": -417.9937438964844,
|
|
"logps/weighted_chosen": -2.5140380859375,
|
|
"logps/weighted_rejected": -3.9316039085388184,
|
|
"loss": 0.5611,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -114.50508117675781,
|
|
"rewards/margins": 41.392189025878906,
|
|
"rewards/rejected": -155.9523468017578,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.5683807134628296,
|
|
"rewards/weighted_margins": 0.804516613483429,
|
|
"rewards/weighted_rejected": -1.373052954673767,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.4553781732530751,
|
|
"grad_norm": 23.082002639770508,
|
|
"learning_rate": 6.636909776321128e-07,
|
|
"logits/chosen": -0.8063064813613892,
|
|
"logits/rejected": -0.802105724811554,
|
|
"logps/chosen": -369.12811279296875,
|
|
"logps/rejected": -413.70782470703125,
|
|
"logps/weighted_chosen": -2.945758104324341,
|
|
"logps/weighted_rejected": -3.6049561500549316,
|
|
"loss": 0.5946,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -107.4222640991211,
|
|
"rewards/margins": 41.763671875,
|
|
"rewards/rejected": -149.2078094482422,
|
|
"rewards/weighted_accuracies": 0.65625,
|
|
"rewards/weighted_chosen": -0.640515148639679,
|
|
"rewards/weighted_margins": 0.601641833782196,
|
|
"rewards/weighted_rejected": -1.2423064708709717,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.46061240512954726,
|
|
"grad_norm": 27.672487258911133,
|
|
"learning_rate": 6.550298408701174e-07,
|
|
"logits/chosen": -0.778796374797821,
|
|
"logits/rejected": -0.830426037311554,
|
|
"logps/chosen": -389.8421936035156,
|
|
"logps/rejected": -428.6312561035156,
|
|
"logps/weighted_chosen": -3.13909912109375,
|
|
"logps/weighted_rejected": -4.012915134429932,
|
|
"loss": 0.6358,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -103.14042663574219,
|
|
"rewards/margins": 43.176368713378906,
|
|
"rewards/rejected": -146.2609405517578,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.5248870849609375,
|
|
"rewards/weighted_margins": 0.555926501750946,
|
|
"rewards/weighted_rejected": -1.0807831287384033,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.46584663700601936,
|
|
"grad_norm": 20.153644561767578,
|
|
"learning_rate": 6.463169254006276e-07,
|
|
"logits/chosen": -0.7750915288925171,
|
|
"logits/rejected": -0.8219833374023438,
|
|
"logps/chosen": -377.8421936035156,
|
|
"logps/rejected": -378.10467529296875,
|
|
"logps/weighted_chosen": -2.7843995094299316,
|
|
"logps/weighted_rejected": -3.576098680496216,
|
|
"loss": 0.5705,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -96.46601867675781,
|
|
"rewards/margins": 34.888282775878906,
|
|
"rewards/rejected": -131.3464813232422,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.39057618379592896,
|
|
"rewards/weighted_margins": 0.706072986125946,
|
|
"rewards/weighted_rejected": -1.0967223644256592,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.4710808688824915,
|
|
"grad_norm": 42.77175521850586,
|
|
"learning_rate": 6.375551412666326e-07,
|
|
"logits/chosen": -0.7759063839912415,
|
|
"logits/rejected": -0.8005096316337585,
|
|
"logps/chosen": -379.19219970703125,
|
|
"logps/rejected": -388.72344970703125,
|
|
"logps/weighted_chosen": -2.5501952171325684,
|
|
"logps/weighted_rejected": -3.5579466819763184,
|
|
"loss": 0.6503,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -105.25859069824219,
|
|
"rewards/margins": 27.150781631469727,
|
|
"rewards/rejected": -132.42733764648438,
|
|
"rewards/weighted_accuracies": 0.6312500238418579,
|
|
"rewards/weighted_chosen": -0.5587005615234375,
|
|
"rewards/weighted_margins": 0.5813232660293579,
|
|
"rewards/weighted_rejected": -1.1405792236328125,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.4763151007589636,
|
|
"grad_norm": 24.800992965698242,
|
|
"learning_rate": 6.287474148328583e-07,
|
|
"logits/chosen": -0.7202819585800171,
|
|
"logits/rejected": -0.7240753173828125,
|
|
"logps/chosen": -371.1812438964844,
|
|
"logps/rejected": -371.109375,
|
|
"logps/weighted_chosen": -2.950915575027466,
|
|
"logps/weighted_rejected": -4.175073146820068,
|
|
"loss": 0.6282,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -100.2894515991211,
|
|
"rewards/margins": 23.904687881469727,
|
|
"rewards/rejected": -124.20625305175781,
|
|
"rewards/weighted_accuracies": 0.6656249761581421,
|
|
"rewards/weighted_chosen": -0.5755615234375,
|
|
"rewards/weighted_margins": 0.5273803472518921,
|
|
"rewards/weighted_rejected": -1.1032683849334717,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.48154933263543576,
|
|
"grad_norm": 18.644733428955078,
|
|
"learning_rate": 6.198966878083857e-07,
|
|
"logits/chosen": -0.7572265863418579,
|
|
"logits/rejected": -0.7787246704101562,
|
|
"logps/chosen": -368.4359436035156,
|
|
"logps/rejected": -402.46563720703125,
|
|
"logps/weighted_chosen": -2.8515868186950684,
|
|
"logps/weighted_rejected": -3.4952635765075684,
|
|
"loss": 0.6159,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -97.32890319824219,
|
|
"rewards/margins": 38.184959411621094,
|
|
"rewards/rejected": -135.45703125,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.4532226622104645,
|
|
"rewards/weighted_margins": 0.559436023235321,
|
|
"rewards/weighted_rejected": -1.012457251548767,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.48678356451190785,
|
|
"grad_norm": 44.07575988769531,
|
|
"learning_rate": 6.110059162641439e-07,
|
|
"logits/chosen": -0.7723480463027954,
|
|
"logits/rejected": -0.802471935749054,
|
|
"logps/chosen": -375.984375,
|
|
"logps/rejected": -391.7906188964844,
|
|
"logps/weighted_chosen": -2.391467332839966,
|
|
"logps/weighted_rejected": -3.1367430686950684,
|
|
"loss": 0.6244,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -93.6128921508789,
|
|
"rewards/margins": 30.975391387939453,
|
|
"rewards/rejected": -124.58906555175781,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.4246048033237457,
|
|
"rewards/weighted_margins": 0.47894287109375,
|
|
"rewards/weighted_rejected": -0.904034435749054,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.49201779638838,
|
|
"grad_norm": 26.321582794189453,
|
|
"learning_rate": 6.020780696456059e-07,
|
|
"logits/chosen": -0.7484909296035767,
|
|
"logits/rejected": -0.7926574945449829,
|
|
"logps/chosen": -359.21875,
|
|
"logps/rejected": -411.1890563964844,
|
|
"logps/weighted_chosen": -2.2330689430236816,
|
|
"logps/weighted_rejected": -3.3442625999450684,
|
|
"loss": 0.5653,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -88.5511703491211,
|
|
"rewards/margins": 55.419921875,
|
|
"rewards/rejected": -143.99453735351562,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.47887879610061646,
|
|
"rewards/weighted_margins": 0.6291259527206421,
|
|
"rewards/weighted_rejected": -1.1078612804412842,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.49725202826485215,
|
|
"grad_norm": 44.13637924194336,
|
|
"learning_rate": 5.931161297810185e-07,
|
|
"logits/chosen": -0.8126861453056335,
|
|
"logits/rejected": -0.829357922077179,
|
|
"logps/chosen": -376.6875,
|
|
"logps/rejected": -399.51251220703125,
|
|
"logps/weighted_chosen": -3.1253294944763184,
|
|
"logps/weighted_rejected": -3.8741211891174316,
|
|
"loss": 0.6461,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -103.4652328491211,
|
|
"rewards/margins": 34.713279724121094,
|
|
"rewards/rejected": -138.1457061767578,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.616656482219696,
|
|
"rewards/weighted_margins": 0.5488006472587585,
|
|
"rewards/weighted_rejected": -1.1659362316131592,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.5024862601413242,
|
|
"grad_norm": 24.634550094604492,
|
|
"learning_rate": 5.841230898854959e-07,
|
|
"logits/chosen": -0.742846667766571,
|
|
"logits/rejected": -0.765765368938446,
|
|
"logps/chosen": -421.84686279296875,
|
|
"logps/rejected": -429.4312438964844,
|
|
"logps/weighted_chosen": -2.9749999046325684,
|
|
"logps/weighted_rejected": -3.778076171875,
|
|
"loss": 0.6955,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -121.6539077758789,
|
|
"rewards/margins": 46.04961013793945,
|
|
"rewards/rejected": -167.69375610351562,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": -0.820935070514679,
|
|
"rewards/weighted_margins": 0.548413097858429,
|
|
"rewards/weighted_rejected": -1.369299292564392,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.5077204920177963,
|
|
"grad_norm": 28.748939514160156,
|
|
"learning_rate": 5.751019535613102e-07,
|
|
"logits/chosen": -0.6985992193222046,
|
|
"logits/rejected": -0.7225921750068665,
|
|
"logps/chosen": -365.7984313964844,
|
|
"logps/rejected": -400.90625,
|
|
"logps/weighted_chosen": -2.9286131858825684,
|
|
"logps/weighted_rejected": -4.021093845367432,
|
|
"loss": 0.6325,
|
|
"rewards/accuracies": 0.684374988079071,
|
|
"rewards/chosen": -106.0390625,
|
|
"rewards/margins": 47.953514099121094,
|
|
"rewards/rejected": -153.97265625,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.7157821655273438,
|
|
"rewards/weighted_margins": 0.7237914800643921,
|
|
"rewards/weighted_rejected": -1.439599633216858,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.5129547238942685,
|
|
"grad_norm": 15.98474407196045,
|
|
"learning_rate": 5.660557337947117e-07,
|
|
"logits/chosen": -0.6841033697128296,
|
|
"logits/rejected": -0.6997619867324829,
|
|
"logps/chosen": -409.1937561035156,
|
|
"logps/rejected": -406.05938720703125,
|
|
"logps/weighted_chosen": -2.480639696121216,
|
|
"logps/weighted_rejected": -3.3584961891174316,
|
|
"loss": 0.5997,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -114.5328140258789,
|
|
"rewards/margins": 34.099998474121094,
|
|
"rewards/rejected": -148.6570281982422,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.572741687297821,
|
|
"rewards/weighted_margins": 0.527575671672821,
|
|
"rewards/weighted_rejected": -1.100128173828125,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.5181889557707406,
|
|
"grad_norm": 41.099185943603516,
|
|
"learning_rate": 5.569874519496174e-07,
|
|
"logits/chosen": -0.7119758725166321,
|
|
"logits/rejected": -0.7671966552734375,
|
|
"logps/chosen": -381.44842529296875,
|
|
"logps/rejected": -410.5015563964844,
|
|
"logps/weighted_chosen": -2.8494019508361816,
|
|
"logps/weighted_rejected": -3.8323974609375,
|
|
"loss": 0.6259,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -106.146484375,
|
|
"rewards/margins": 37.33867263793945,
|
|
"rewards/rejected": -143.4011688232422,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.4599502682685852,
|
|
"rewards/weighted_margins": 0.6096404790878296,
|
|
"rewards/weighted_rejected": -1.070257544517517,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"grad_norm": 34.9498176574707,
|
|
"learning_rate": 5.47900136758499e-07,
|
|
"logits/chosen": -0.6499813199043274,
|
|
"logits/rejected": -0.7194549441337585,
|
|
"logps/chosen": -369.4429626464844,
|
|
"logps/rejected": -382.0953063964844,
|
|
"logps/weighted_chosen": -2.7041993141174316,
|
|
"logps/weighted_rejected": -3.534008741378784,
|
|
"loss": 0.5974,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -101.8080062866211,
|
|
"rewards/margins": 38.08086013793945,
|
|
"rewards/rejected": -139.9406280517578,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.44673460721969604,
|
|
"rewards/weighted_margins": 0.6419677734375,
|
|
"rewards/weighted_rejected": -1.08880615234375,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"eval_logits/chosen": -0.7614516615867615,
|
|
"eval_logits/rejected": -0.7845029234886169,
|
|
"eval_logps/chosen": -403.2145080566406,
|
|
"eval_logps/rejected": -419.8420104980469,
|
|
"eval_logps/weighted_chosen": -2.8744430541992188,
|
|
"eval_logps/weighted_rejected": -3.6894454956054688,
|
|
"eval_loss": 0.6146492958068848,
|
|
"eval_rewards/accuracies": 0.6269999742507935,
|
|
"eval_rewards/chosen": -114.45649719238281,
|
|
"eval_rewards/margins": 37.66427993774414,
|
|
"eval_rewards/rejected": -152.1232452392578,
|
|
"eval_rewards/weighted_accuracies": 0.6679999828338623,
|
|
"eval_rewards/weighted_chosen": -0.6033662557601929,
|
|
"eval_rewards/weighted_margins": 0.5922585129737854,
|
|
"eval_rewards/weighted_rejected": -1.195624828338623,
|
|
"eval_runtime": 1076.2039,
|
|
"eval_samples_per_second": 1.858,
|
|
"eval_steps_per_second": 0.465,
|
|
"step": 1000
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1911,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|