2200 lines
86 KiB
JSON
2200 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5234231876472127,
|
|
"eval_steps": 500,
|
|
"global_step": 1000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0005234231876472127,
|
|
"grad_norm": 132.6717987060547,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.40118408203125,
|
|
"logits/rejected": -0.41802978515625,
|
|
"logps/chosen": -297.609375,
|
|
"logps/rejected": -247.84375,
|
|
"logps/weighted_chosen": -4.7568359375,
|
|
"logps/weighted_rejected": -3.47998046875,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"rewards/weighted_accuracies": 0.0,
|
|
"rewards/weighted_chosen": 0.0,
|
|
"rewards/weighted_margins": 0.0,
|
|
"rewards/weighted_rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.005234231876472127,
|
|
"grad_norm": 226.00839233398438,
|
|
"learning_rate": 4.6875e-08,
|
|
"logits/chosen": -0.3175845742225647,
|
|
"logits/rejected": -0.3532341718673706,
|
|
"logps/chosen": -275.5841979980469,
|
|
"logps/rejected": -255.84548950195312,
|
|
"logps/weighted_chosen": -2.651665687561035,
|
|
"logps/weighted_rejected": -2.88427734375,
|
|
"loss": 0.6921,
|
|
"rewards/accuracies": 0.25,
|
|
"rewards/chosen": -0.0731336772441864,
|
|
"rewards/margins": -0.0670572891831398,
|
|
"rewards/rejected": -0.006076388992369175,
|
|
"rewards/weighted_accuracies": 0.2951388955116272,
|
|
"rewards/weighted_chosen": -0.0015835232334211469,
|
|
"rewards/weighted_margins": -0.0009324815473519266,
|
|
"rewards/weighted_rejected": -0.0006510416860692203,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.010468463752944255,
|
|
"grad_norm": 14.726158142089844,
|
|
"learning_rate": 9.895833333333332e-08,
|
|
"logits/chosen": -0.29781341552734375,
|
|
"logits/rejected": -0.3136836886405945,
|
|
"logps/chosen": -294.4296875,
|
|
"logps/rejected": -272.58984375,
|
|
"logps/weighted_chosen": -2.458728075027466,
|
|
"logps/weighted_rejected": -2.455883741378784,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.30000001192092896,
|
|
"rewards/chosen": -0.14013671875,
|
|
"rewards/margins": -0.13369140028953552,
|
|
"rewards/rejected": -0.0064453124068677425,
|
|
"rewards/weighted_accuracies": 0.34062498807907104,
|
|
"rewards/weighted_chosen": -0.0013603210682049394,
|
|
"rewards/weighted_margins": -0.0010925292735919356,
|
|
"rewards/weighted_rejected": -0.0002677917364053428,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.015702695629416383,
|
|
"grad_norm": 77.6125259399414,
|
|
"learning_rate": 1.5104166666666664e-07,
|
|
"logits/chosen": -0.2918853759765625,
|
|
"logits/rejected": -0.3377639651298523,
|
|
"logps/chosen": -298.05859375,
|
|
"logps/rejected": -268.0132751464844,
|
|
"logps/weighted_chosen": -2.4350829124450684,
|
|
"logps/weighted_rejected": -2.7343993186950684,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.24062499403953552,
|
|
"rewards/chosen": -0.06621094048023224,
|
|
"rewards/margins": -0.1640625,
|
|
"rewards/rejected": 0.09785155951976776,
|
|
"rewards/weighted_accuracies": 0.3187499940395355,
|
|
"rewards/weighted_chosen": 0.0014068603049963713,
|
|
"rewards/weighted_margins": -0.0015777588123455644,
|
|
"rewards/weighted_rejected": 0.0029846192337572575,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.02093692750588851,
|
|
"grad_norm": 30.666196823120117,
|
|
"learning_rate": 2.03125e-07,
|
|
"logits/chosen": -0.30072021484375,
|
|
"logits/rejected": -0.3433845639228821,
|
|
"logps/chosen": -278.68829345703125,
|
|
"logps/rejected": -253.90780639648438,
|
|
"logps/weighted_chosen": -2.506396532058716,
|
|
"logps/weighted_rejected": -2.8416504859924316,
|
|
"loss": 0.6908,
|
|
"rewards/accuracies": 0.3062500059604645,
|
|
"rewards/chosen": 0.063232421875,
|
|
"rewards/margins": 0.04838867112994194,
|
|
"rewards/rejected": 0.014843749813735485,
|
|
"rewards/weighted_accuracies": 0.40312498807907104,
|
|
"rewards/weighted_chosen": 0.0042968750931322575,
|
|
"rewards/weighted_margins": 0.0019538879860192537,
|
|
"rewards/weighted_rejected": 0.0023429871071130037,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.02617115938236064,
|
|
"grad_norm": 18.60569953918457,
|
|
"learning_rate": 2.552083333333333e-07,
|
|
"logits/chosen": -0.2819870114326477,
|
|
"logits/rejected": -0.32059136033058167,
|
|
"logps/chosen": -280.31951904296875,
|
|
"logps/rejected": -267.4359436035156,
|
|
"logps/weighted_chosen": -2.4267334938049316,
|
|
"logps/weighted_rejected": -2.529711961746216,
|
|
"loss": 0.6891,
|
|
"rewards/accuracies": 0.3187499940395355,
|
|
"rewards/chosen": -0.03535156324505806,
|
|
"rewards/margins": -0.13984374701976776,
|
|
"rewards/rejected": 0.1044921875,
|
|
"rewards/weighted_accuracies": 0.3968749940395355,
|
|
"rewards/weighted_chosen": 0.0039031982887536287,
|
|
"rewards/weighted_margins": 0.005755615420639515,
|
|
"rewards/weighted_rejected": -0.0018524170154705644,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.031405391258832765,
|
|
"grad_norm": 38.21036911010742,
|
|
"learning_rate": 3.0729166666666665e-07,
|
|
"logits/chosen": -0.31453245878219604,
|
|
"logits/rejected": -0.30809077620506287,
|
|
"logps/chosen": -277.66015625,
|
|
"logps/rejected": -261.7445373535156,
|
|
"logps/weighted_chosen": -2.8622069358825684,
|
|
"logps/weighted_rejected": -2.7553467750549316,
|
|
"loss": 0.6894,
|
|
"rewards/accuracies": 0.35624998807907104,
|
|
"rewards/chosen": 0.04150390625,
|
|
"rewards/margins": 0.08027343451976776,
|
|
"rewards/rejected": -0.03876953199505806,
|
|
"rewards/weighted_accuracies": 0.4312500059604645,
|
|
"rewards/weighted_chosen": 0.0006561279296875,
|
|
"rewards/weighted_margins": 0.006243896670639515,
|
|
"rewards/weighted_rejected": -0.005587768740952015,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.036639623135304895,
|
|
"grad_norm": 69.19047546386719,
|
|
"learning_rate": 3.59375e-07,
|
|
"logits/chosen": -0.3177490234375,
|
|
"logits/rejected": -0.3246749937534332,
|
|
"logps/chosen": -289.76251220703125,
|
|
"logps/rejected": -244.92578125,
|
|
"logps/weighted_chosen": -2.3438963890075684,
|
|
"logps/weighted_rejected": -2.7010498046875,
|
|
"loss": 0.6841,
|
|
"rewards/accuracies": 0.49687498807907104,
|
|
"rewards/chosen": 0.29765623807907104,
|
|
"rewards/margins": 0.4546875059604645,
|
|
"rewards/rejected": -0.15703125298023224,
|
|
"rewards/weighted_accuracies": 0.5406249761581421,
|
|
"rewards/weighted_chosen": 0.01530532818287611,
|
|
"rewards/weighted_margins": 0.01918792724609375,
|
|
"rewards/weighted_rejected": -0.0038825988303869963,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.04187385501177702,
|
|
"grad_norm": 51.98476791381836,
|
|
"learning_rate": 4.114583333333333e-07,
|
|
"logits/chosen": -0.2850998044013977,
|
|
"logits/rejected": -0.30662041902542114,
|
|
"logps/chosen": -289.234375,
|
|
"logps/rejected": -270.375,
|
|
"logps/weighted_chosen": -2.5325684547424316,
|
|
"logps/weighted_rejected": -2.796435594558716,
|
|
"loss": 0.6747,
|
|
"rewards/accuracies": 0.5062500238418579,
|
|
"rewards/chosen": 0.512499988079071,
|
|
"rewards/margins": 0.6001952886581421,
|
|
"rewards/rejected": -0.08769531548023224,
|
|
"rewards/weighted_accuracies": 0.5562499761581421,
|
|
"rewards/weighted_chosen": 0.036380767822265625,
|
|
"rewards/weighted_margins": 0.04396667331457138,
|
|
"rewards/weighted_rejected": -0.007586670108139515,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.04710808688824915,
|
|
"grad_norm": 30.52783203125,
|
|
"learning_rate": 4.6354166666666664e-07,
|
|
"logits/chosen": -0.3142959475517273,
|
|
"logits/rejected": -0.3075408935546875,
|
|
"logps/chosen": -280.11407470703125,
|
|
"logps/rejected": -257.95233154296875,
|
|
"logps/weighted_chosen": -2.719482421875,
|
|
"logps/weighted_rejected": -2.88037109375,
|
|
"loss": 0.6687,
|
|
"rewards/accuracies": 0.5062500238418579,
|
|
"rewards/chosen": 0.5205078125,
|
|
"rewards/margins": 0.737109363079071,
|
|
"rewards/rejected": -0.21660156548023224,
|
|
"rewards/weighted_accuracies": 0.621874988079071,
|
|
"rewards/weighted_chosen": 0.06780395656824112,
|
|
"rewards/weighted_margins": 0.07340697944164276,
|
|
"rewards/weighted_rejected": -0.0056396485306322575,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.05234231876472128,
|
|
"grad_norm": 69.397705078125,
|
|
"learning_rate": 5.156249999999999e-07,
|
|
"logits/chosen": -0.28213196992874146,
|
|
"logits/rejected": -0.3543289303779602,
|
|
"logps/chosen": -290.71875,
|
|
"logps/rejected": -286.73126220703125,
|
|
"logps/weighted_chosen": -2.2228636741638184,
|
|
"logps/weighted_rejected": -2.8367552757263184,
|
|
"loss": 0.6848,
|
|
"rewards/accuracies": 0.5718749761581421,
|
|
"rewards/chosen": 0.24521484971046448,
|
|
"rewards/margins": 1.0690429210662842,
|
|
"rewards/rejected": -0.8238281011581421,
|
|
"rewards/weighted_accuracies": 0.5843750238418579,
|
|
"rewards/weighted_chosen": 0.05242309719324112,
|
|
"rewards/weighted_margins": 0.05032653734087944,
|
|
"rewards/weighted_rejected": 0.0021240233909338713,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.05757655064119341,
|
|
"grad_norm": 36.600040435791016,
|
|
"learning_rate": 5.677083333333333e-07,
|
|
"logits/chosen": -0.33063429594039917,
|
|
"logits/rejected": -0.319937139749527,
|
|
"logps/chosen": -296.82501220703125,
|
|
"logps/rejected": -262.2984313964844,
|
|
"logps/weighted_chosen": -2.8468017578125,
|
|
"logps/weighted_rejected": -2.9306397438049316,
|
|
"loss": 0.6773,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -0.474609375,
|
|
"rewards/margins": 1.053613305091858,
|
|
"rewards/rejected": -1.528222680091858,
|
|
"rewards/weighted_accuracies": 0.534375011920929,
|
|
"rewards/weighted_chosen": 0.013439941219985485,
|
|
"rewards/weighted_margins": 0.05541381984949112,
|
|
"rewards/weighted_rejected": -0.04198913648724556,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.06281078251766553,
|
|
"grad_norm": 57.109580993652344,
|
|
"learning_rate": 6.197916666666666e-07,
|
|
"logits/chosen": -0.33633461594581604,
|
|
"logits/rejected": -0.36155110597610474,
|
|
"logps/chosen": -295.3687438964844,
|
|
"logps/rejected": -256.1953125,
|
|
"logps/weighted_chosen": -2.161865234375,
|
|
"logps/weighted_rejected": -2.4251465797424316,
|
|
"loss": 0.6791,
|
|
"rewards/accuracies": 0.5718749761581421,
|
|
"rewards/chosen": -0.72900390625,
|
|
"rewards/margins": 1.641210913658142,
|
|
"rewards/rejected": -2.3702149391174316,
|
|
"rewards/weighted_accuracies": 0.5562499761581421,
|
|
"rewards/weighted_chosen": 0.007176590152084827,
|
|
"rewards/weighted_margins": 0.05286560207605362,
|
|
"rewards/weighted_rejected": -0.04570160061120987,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.06804501439413765,
|
|
"grad_norm": 39.176841735839844,
|
|
"learning_rate": 6.718749999999999e-07,
|
|
"logits/chosen": -0.29625242948532104,
|
|
"logits/rejected": -0.2914108335971832,
|
|
"logps/chosen": -306.6781311035156,
|
|
"logps/rejected": -280.15936279296875,
|
|
"logps/weighted_chosen": -2.188079833984375,
|
|
"logps/weighted_rejected": -2.5787596702575684,
|
|
"loss": 0.6659,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -0.644238293170929,
|
|
"rewards/margins": 1.972265601158142,
|
|
"rewards/rejected": -2.616406202316284,
|
|
"rewards/weighted_accuracies": 0.606249988079071,
|
|
"rewards/weighted_chosen": 0.01349639892578125,
|
|
"rewards/weighted_margins": 0.0841522216796875,
|
|
"rewards/weighted_rejected": -0.07064209133386612,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.07327924627060979,
|
|
"grad_norm": 52.14993667602539,
|
|
"learning_rate": 7.239583333333333e-07,
|
|
"logits/chosen": -0.3304199278354645,
|
|
"logits/rejected": -0.3464847505092621,
|
|
"logps/chosen": -301.4390563964844,
|
|
"logps/rejected": -277.9515686035156,
|
|
"logps/weighted_chosen": -2.554003953933716,
|
|
"logps/weighted_rejected": -2.881591796875,
|
|
"loss": 0.6581,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -2.746875047683716,
|
|
"rewards/margins": 2.744921922683716,
|
|
"rewards/rejected": -5.491991996765137,
|
|
"rewards/weighted_accuracies": 0.6312500238418579,
|
|
"rewards/weighted_chosen": -0.02762756310403347,
|
|
"rewards/weighted_margins": 0.11510010063648224,
|
|
"rewards/weighted_rejected": -0.14276733994483948,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.07851347814708191,
|
|
"grad_norm": 22.611814498901367,
|
|
"learning_rate": 7.760416666666666e-07,
|
|
"logits/chosen": -0.2870376706123352,
|
|
"logits/rejected": -0.2975311279296875,
|
|
"logps/chosen": -287.859375,
|
|
"logps/rejected": -257.54296875,
|
|
"logps/weighted_chosen": -3.089892625808716,
|
|
"logps/weighted_rejected": -3.1946043968200684,
|
|
"loss": 0.6544,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -3.7095704078674316,
|
|
"rewards/margins": 2.942578077316284,
|
|
"rewards/rejected": -6.652148246765137,
|
|
"rewards/weighted_accuracies": 0.6312500238418579,
|
|
"rewards/weighted_chosen": -0.005145263858139515,
|
|
"rewards/weighted_margins": 0.16416625678539276,
|
|
"rewards/weighted_rejected": -0.16951599717140198,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.08374771002355404,
|
|
"grad_norm": 15.511767387390137,
|
|
"learning_rate": 8.28125e-07,
|
|
"logits/chosen": -0.3232177793979645,
|
|
"logits/rejected": -0.3726806640625,
|
|
"logps/chosen": -308.91796875,
|
|
"logps/rejected": -282.15704345703125,
|
|
"logps/weighted_chosen": -2.5903563499450684,
|
|
"logps/weighted_rejected": -2.742602586746216,
|
|
"loss": 0.6211,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -4.956835746765137,
|
|
"rewards/margins": 3.9961915016174316,
|
|
"rewards/rejected": -8.953222274780273,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.002410888671875,
|
|
"rewards/weighted_margins": 0.23797607421875,
|
|
"rewards/weighted_rejected": -0.24028320610523224,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.08898194190002617,
|
|
"grad_norm": 167.33956909179688,
|
|
"learning_rate": 8.802083333333333e-07,
|
|
"logits/chosen": -0.36021536588668823,
|
|
"logits/rejected": -0.3597045838832855,
|
|
"logps/chosen": -311.03045654296875,
|
|
"logps/rejected": -270.46875,
|
|
"logps/weighted_chosen": -2.8318848609924316,
|
|
"logps/weighted_rejected": -3.139453172683716,
|
|
"loss": 0.6949,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -8.0087890625,
|
|
"rewards/margins": 4.345898628234863,
|
|
"rewards/rejected": -12.354199409484863,
|
|
"rewards/weighted_accuracies": 0.640625,
|
|
"rewards/weighted_chosen": -0.01859130896627903,
|
|
"rewards/weighted_margins": 0.20853272080421448,
|
|
"rewards/weighted_rejected": -0.22731323540210724,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.0942161737764983,
|
|
"grad_norm": 64.57138061523438,
|
|
"learning_rate": 9.322916666666666e-07,
|
|
"logits/chosen": -0.33618468046188354,
|
|
"logits/rejected": -0.3534431457519531,
|
|
"logps/chosen": -284.2171936035156,
|
|
"logps/rejected": -272.12969970703125,
|
|
"logps/weighted_chosen": -2.694580078125,
|
|
"logps/weighted_rejected": -3.225878953933716,
|
|
"loss": 0.6814,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -10.43701171875,
|
|
"rewards/margins": 5.353320121765137,
|
|
"rewards/rejected": -15.7919921875,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": -0.08297424018383026,
|
|
"rewards/weighted_margins": 0.26459962129592896,
|
|
"rewards/weighted_rejected": -0.347381591796875,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.09945040565297043,
|
|
"grad_norm": 49.0852165222168,
|
|
"learning_rate": 9.84375e-07,
|
|
"logits/chosen": -0.354086309671402,
|
|
"logits/rejected": -0.38891831040382385,
|
|
"logps/chosen": -319.17498779296875,
|
|
"logps/rejected": -283.31561279296875,
|
|
"logps/weighted_chosen": -2.5078492164611816,
|
|
"logps/weighted_rejected": -3.016357421875,
|
|
"loss": 0.6496,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -13.561426162719727,
|
|
"rewards/margins": 5.937890529632568,
|
|
"rewards/rejected": -19.498828887939453,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.16942748427391052,
|
|
"rewards/weighted_margins": 0.24410399794578552,
|
|
"rewards/weighted_rejected": -0.41356199979782104,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.10468463752944256,
|
|
"grad_norm": 53.46296691894531,
|
|
"learning_rate": 9.99959085414323e-07,
|
|
"logits/chosen": -0.37868577241897583,
|
|
"logits/rejected": -0.4114578366279602,
|
|
"logps/chosen": -324.7124938964844,
|
|
"logps/rejected": -279.72967529296875,
|
|
"logps/weighted_chosen": -2.8757567405700684,
|
|
"logps/weighted_rejected": -3.3623046875,
|
|
"loss": 0.639,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -15.428125381469727,
|
|
"rewards/margins": 6.552148342132568,
|
|
"rewards/rejected": -21.975000381469727,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.18135985732078552,
|
|
"rewards/weighted_margins": 0.29008787870407104,
|
|
"rewards/weighted_rejected": -0.471527099609375,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.10991886940591468,
|
|
"grad_norm": 24.815481185913086,
|
|
"learning_rate": 9.997587035630105e-07,
|
|
"logits/chosen": -0.3853309750556946,
|
|
"logits/rejected": -0.4257049560546875,
|
|
"logps/chosen": -302.82891845703125,
|
|
"logps/rejected": -308.671875,
|
|
"logps/weighted_chosen": -2.632519483566284,
|
|
"logps/weighted_rejected": -3.3669190406799316,
|
|
"loss": 0.6558,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -15.814453125,
|
|
"rewards/margins": 8.331445693969727,
|
|
"rewards/rejected": -24.146093368530273,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.225901797413826,
|
|
"rewards/weighted_margins": 0.23236694931983948,
|
|
"rewards/weighted_rejected": -0.45829468965530396,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.11515310128238682,
|
|
"grad_norm": 24.175745010375977,
|
|
"learning_rate": 9.99391406364405e-07,
|
|
"logits/chosen": -0.37365952134132385,
|
|
"logits/rejected": -0.3758789002895355,
|
|
"logps/chosen": -309.34686279296875,
|
|
"logps/rejected": -293.98126220703125,
|
|
"logps/weighted_chosen": -3.002514600753784,
|
|
"logps/weighted_rejected": -3.453906297683716,
|
|
"loss": 0.6732,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -17.203418731689453,
|
|
"rewards/margins": 7.933203220367432,
|
|
"rewards/rejected": -25.137109756469727,
|
|
"rewards/weighted_accuracies": 0.6156250238418579,
|
|
"rewards/weighted_chosen": -0.259225457906723,
|
|
"rewards/weighted_margins": 0.29540252685546875,
|
|
"rewards/weighted_rejected": -0.5546798706054688,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.12038733315885894,
|
|
"grad_norm": 85.15988159179688,
|
|
"learning_rate": 9.988573164927884e-07,
|
|
"logits/chosen": -0.3097473084926605,
|
|
"logits/rejected": -0.3477935791015625,
|
|
"logps/chosen": -286.5078125,
|
|
"logps/rejected": -281.8453063964844,
|
|
"logps/weighted_chosen": -2.66943359375,
|
|
"logps/weighted_rejected": -3.1229491233825684,
|
|
"loss": 0.6646,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -17.690723419189453,
|
|
"rewards/margins": 12.424609184265137,
|
|
"rewards/rejected": -30.110157012939453,
|
|
"rewards/weighted_accuracies": 0.65625,
|
|
"rewards/weighted_chosen": -0.2716217041015625,
|
|
"rewards/weighted_margins": 0.32661741971969604,
|
|
"rewards/weighted_rejected": -0.5983597040176392,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.12562156503533106,
|
|
"grad_norm": 26.17377471923828,
|
|
"learning_rate": 9.98156612329838e-07,
|
|
"logits/chosen": -0.39516907930374146,
|
|
"logits/rejected": -0.44511109590530396,
|
|
"logps/chosen": -286.74884033203125,
|
|
"logps/rejected": -318.22735595703125,
|
|
"logps/weighted_chosen": -2.6696534156799316,
|
|
"logps/weighted_rejected": -3.4151854515075684,
|
|
"loss": 0.643,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -21.990428924560547,
|
|
"rewards/margins": 14.028905868530273,
|
|
"rewards/rejected": -36.013282775878906,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.2329559326171875,
|
|
"rewards/weighted_margins": 0.3950134217739105,
|
|
"rewards/weighted_rejected": -0.6281493902206421,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.13085579691180318,
|
|
"grad_norm": 56.73057174682617,
|
|
"learning_rate": 9.97289527905053e-07,
|
|
"logits/chosen": -0.40631332993507385,
|
|
"logits/rejected": -0.4203124940395355,
|
|
"logps/chosen": -290.1703186035156,
|
|
"logps/rejected": -291.6328125,
|
|
"logps/weighted_chosen": -3.051513671875,
|
|
"logps/weighted_rejected": -3.3163819313049316,
|
|
"loss": 0.677,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -25.742870330810547,
|
|
"rewards/margins": 9.973828315734863,
|
|
"rewards/rejected": -35.72148513793945,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": -0.2856689393520355,
|
|
"rewards/weighted_margins": 0.253326416015625,
|
|
"rewards/weighted_rejected": -0.5388733148574829,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.1360900287882753,
|
|
"grad_norm": 17.766258239746094,
|
|
"learning_rate": 9.962563528175875e-07,
|
|
"logits/chosen": -0.3611465394496918,
|
|
"logits/rejected": -0.39628905057907104,
|
|
"logps/chosen": -324.36639404296875,
|
|
"logps/rejected": -297.765625,
|
|
"logps/weighted_chosen": -2.652392625808716,
|
|
"logps/weighted_rejected": -3.535571336746216,
|
|
"loss": 0.6414,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -25.621288299560547,
|
|
"rewards/margins": 11.306055068969727,
|
|
"rewards/rejected": -36.93359375,
|
|
"rewards/weighted_accuracies": 0.637499988079071,
|
|
"rewards/weighted_chosen": -0.2533508241176605,
|
|
"rewards/weighted_margins": 0.2956604063510895,
|
|
"rewards/weighted_rejected": -0.5490142703056335,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.14132426066474746,
|
|
"grad_norm": 17.552453994750977,
|
|
"learning_rate": 9.950574321395277e-07,
|
|
"logits/chosen": -0.41735154390335083,
|
|
"logits/rejected": -0.441476434469223,
|
|
"logps/chosen": -314.5093688964844,
|
|
"logps/rejected": -295.7093811035156,
|
|
"logps/weighted_chosen": -2.864941358566284,
|
|
"logps/weighted_rejected": -3.25732421875,
|
|
"loss": 0.661,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -25.1123046875,
|
|
"rewards/margins": 7.519726753234863,
|
|
"rewards/rejected": -32.62890625,
|
|
"rewards/weighted_accuracies": 0.609375,
|
|
"rewards/weighted_chosen": -0.3035888671875,
|
|
"rewards/weighted_margins": 0.2833190858364105,
|
|
"rewards/weighted_rejected": -0.5868393182754517,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.14655849254121958,
|
|
"grad_norm": 47.66518020629883,
|
|
"learning_rate": 9.936931663006413e-07,
|
|
"logits/chosen": -0.4760284423828125,
|
|
"logits/rejected": -0.46795654296875,
|
|
"logps/chosen": -323.48126220703125,
|
|
"logps/rejected": -313.2875061035156,
|
|
"logps/weighted_chosen": -2.794970750808716,
|
|
"logps/weighted_rejected": -3.3581910133361816,
|
|
"loss": 0.6169,
|
|
"rewards/accuracies": 0.690625011920929,
|
|
"rewards/chosen": -20.707616806030273,
|
|
"rewards/margins": 13.166601181030273,
|
|
"rewards/rejected": -33.86640548706055,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.10174255073070526,
|
|
"rewards/weighted_margins": 0.34544676542282104,
|
|
"rewards/weighted_rejected": -0.447021484375,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.1517927244176917,
|
|
"grad_norm": 32.503883361816406,
|
|
"learning_rate": 9.921640109546357e-07,
|
|
"logits/chosen": -0.44742050766944885,
|
|
"logits/rejected": -0.5166229009628296,
|
|
"logps/chosen": -292.1796875,
|
|
"logps/rejected": -289.6234436035156,
|
|
"logps/weighted_chosen": -2.7469239234924316,
|
|
"logps/weighted_rejected": -3.9541258811950684,
|
|
"loss": 0.6249,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -25.293359756469727,
|
|
"rewards/margins": 12.698633193969727,
|
|
"rewards/rejected": -37.994529724121094,
|
|
"rewards/weighted_accuracies": 0.628125011920929,
|
|
"rewards/weighted_chosen": -0.15215758979320526,
|
|
"rewards/weighted_margins": 0.4393859803676605,
|
|
"rewards/weighted_rejected": -0.5915588140487671,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.15702695629416383,
|
|
"grad_norm": 17.32170867919922,
|
|
"learning_rate": 9.90470476826975e-07,
|
|
"logits/chosen": -0.5146636962890625,
|
|
"logits/rejected": -0.515917956829071,
|
|
"logps/chosen": -302.3570251464844,
|
|
"logps/rejected": -313.68438720703125,
|
|
"logps/weighted_chosen": -2.6830201148986816,
|
|
"logps/weighted_rejected": -3.202099561691284,
|
|
"loss": 0.6526,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -32.978126525878906,
|
|
"rewards/margins": 13.435937881469727,
|
|
"rewards/rejected": -46.408203125,
|
|
"rewards/weighted_accuracies": 0.621874988079071,
|
|
"rewards/weighted_chosen": -0.23505249619483948,
|
|
"rewards/weighted_margins": 0.33623045682907104,
|
|
"rewards/weighted_rejected": -0.5710296630859375,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.16226118817063595,
|
|
"grad_norm": 25.855854034423828,
|
|
"learning_rate": 9.886131295443002e-07,
|
|
"logits/chosen": -0.6332122683525085,
|
|
"logits/rejected": -0.6879852414131165,
|
|
"logps/chosen": -315.02264404296875,
|
|
"logps/rejected": -296.54998779296875,
|
|
"logps/weighted_chosen": -2.8891844749450684,
|
|
"logps/weighted_rejected": -3.3497071266174316,
|
|
"loss": 0.6099,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -33.83808517456055,
|
|
"rewards/margins": 12.542577743530273,
|
|
"rewards/rejected": -46.39081954956055,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.20775146782398224,
|
|
"rewards/weighted_margins": 0.507794201374054,
|
|
"rewards/weighted_rejected": -0.715728759765625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.16749542004710807,
|
|
"grad_norm": 19.11484718322754,
|
|
"learning_rate": 9.865925894455166e-07,
|
|
"logits/chosen": -0.730267345905304,
|
|
"logits/rejected": -0.746167004108429,
|
|
"logps/chosen": -338.2242126464844,
|
|
"logps/rejected": -307.18280029296875,
|
|
"logps/weighted_chosen": -2.9883790016174316,
|
|
"logps/weighted_rejected": -3.5892090797424316,
|
|
"loss": 0.6942,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -38.233009338378906,
|
|
"rewards/margins": 11.564062118530273,
|
|
"rewards/rejected": -49.80937576293945,
|
|
"rewards/weighted_accuracies": 0.612500011920929,
|
|
"rewards/weighted_chosen": -0.3507080078125,
|
|
"rewards/weighted_margins": 0.3366760313510895,
|
|
"rewards/weighted_rejected": -0.6871337890625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.17272965192358022,
|
|
"grad_norm": 57.19697570800781,
|
|
"learning_rate": 9.84409531374603e-07,
|
|
"logits/chosen": -0.6843910217285156,
|
|
"logits/rejected": -0.6659576296806335,
|
|
"logps/chosen": -345.46875,
|
|
"logps/rejected": -316.2515563964844,
|
|
"logps/weighted_chosen": -3.05517578125,
|
|
"logps/weighted_rejected": -3.5519776344299316,
|
|
"loss": 0.6569,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -40.732032775878906,
|
|
"rewards/margins": 12.651952743530273,
|
|
"rewards/rejected": -53.38984298706055,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.3262878358364105,
|
|
"rewards/weighted_margins": 0.346893310546875,
|
|
"rewards/weighted_rejected": -0.6730865240097046,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.17796388380005235,
|
|
"grad_norm": 52.49288558959961,
|
|
"learning_rate": 9.820646844552219e-07,
|
|
"logits/chosen": -0.6993133425712585,
|
|
"logits/rejected": -0.7529846429824829,
|
|
"logps/chosen": -313.59295654296875,
|
|
"logps/rejected": -322.1499938964844,
|
|
"logps/weighted_chosen": -3.0488524436950684,
|
|
"logps/weighted_rejected": -3.440136671066284,
|
|
"loss": 0.6287,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -37.06660079956055,
|
|
"rewards/margins": 19.494531631469727,
|
|
"rewards/rejected": -56.556640625,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.30719298124313354,
|
|
"rewards/weighted_margins": 0.448944091796875,
|
|
"rewards/weighted_rejected": -0.755999743938446,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.18319811567652447,
|
|
"grad_norm": 15.657389640808105,
|
|
"learning_rate": 9.795588318471964e-07,
|
|
"logits/chosen": -0.7813507318496704,
|
|
"logits/rejected": -0.7874206304550171,
|
|
"logps/chosen": -299.80157470703125,
|
|
"logps/rejected": -331.4375,
|
|
"logps/weighted_chosen": -2.84619140625,
|
|
"logps/weighted_rejected": -3.315380811691284,
|
|
"loss": 0.6405,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -40.32304763793945,
|
|
"rewards/margins": 14.830663681030273,
|
|
"rewards/rejected": -55.15625,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.315826416015625,
|
|
"rewards/weighted_margins": 0.386627197265625,
|
|
"rewards/weighted_rejected": -0.702471911907196,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.1884323475529966,
|
|
"grad_norm": 16.19976806640625,
|
|
"learning_rate": 9.768928104849415e-07,
|
|
"logits/chosen": -0.801177978515625,
|
|
"logits/rejected": -0.799664318561554,
|
|
"logps/chosen": -323.5171813964844,
|
|
"logps/rejected": -305.046875,
|
|
"logps/weighted_chosen": -3.1164307594299316,
|
|
"logps/weighted_rejected": -3.3475098609924316,
|
|
"loss": 0.6865,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -40.823829650878906,
|
|
"rewards/margins": 15.389843940734863,
|
|
"rewards/rejected": -56.216407775878906,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.31828004121780396,
|
|
"rewards/weighted_margins": 0.3831420838832855,
|
|
"rewards/weighted_rejected": -0.7014526128768921,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.19366657942946872,
|
|
"grad_norm": 89.87427520751953,
|
|
"learning_rate": 9.740675107979355e-07,
|
|
"logits/chosen": -0.7640800476074219,
|
|
"logits/rejected": -0.7867538332939148,
|
|
"logps/chosen": -361.13751220703125,
|
|
"logps/rejected": -334.97967529296875,
|
|
"logps/weighted_chosen": -2.5084471702575684,
|
|
"logps/weighted_rejected": -3.4689698219299316,
|
|
"loss": 0.6531,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": -46.098045349121094,
|
|
"rewards/margins": 14.188085556030273,
|
|
"rewards/rejected": -60.26640701293945,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.36387938261032104,
|
|
"rewards/weighted_margins": 0.3567260801792145,
|
|
"rewards/weighted_rejected": -0.720538318157196,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.19890081130594087,
|
|
"grad_norm": 22.484216690063477,
|
|
"learning_rate": 9.71083876413323e-07,
|
|
"logits/chosen": -0.7209137082099915,
|
|
"logits/rejected": -0.7318176031112671,
|
|
"logps/chosen": -353.6031188964844,
|
|
"logps/rejected": -339.16485595703125,
|
|
"logps/weighted_chosen": -2.70361328125,
|
|
"logps/weighted_rejected": -3.5843749046325684,
|
|
"loss": 0.6589,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -51.237892150878906,
|
|
"rewards/margins": 18.424999237060547,
|
|
"rewards/rejected": -69.64530944824219,
|
|
"rewards/weighted_accuracies": 0.6312500238418579,
|
|
"rewards/weighted_chosen": -0.4475158751010895,
|
|
"rewards/weighted_margins": 0.3267761170864105,
|
|
"rewards/weighted_rejected": -0.7747405767440796,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.204135043182413,
|
|
"grad_norm": 21.885372161865234,
|
|
"learning_rate": 9.67942903840751e-07,
|
|
"logits/chosen": -0.7708206176757812,
|
|
"logits/rejected": -0.8207153081893921,
|
|
"logps/chosen": -355.18438720703125,
|
|
"logps/rejected": -350.47186279296875,
|
|
"logps/weighted_chosen": -2.8836669921875,
|
|
"logps/weighted_rejected": -3.5904297828674316,
|
|
"loss": 0.6028,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -50.973045349121094,
|
|
"rewards/margins": 25.190038681030273,
|
|
"rewards/rejected": -76.1617202758789,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.39097899198532104,
|
|
"rewards/weighted_margins": 0.4941650331020355,
|
|
"rewards/weighted_rejected": -0.884967029094696,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.2093692750588851,
|
|
"grad_norm": 26.357742309570312,
|
|
"learning_rate": 9.646456421395447e-07,
|
|
"logits/chosen": -0.805267333984375,
|
|
"logits/rejected": -0.8178039789199829,
|
|
"logps/chosen": -377.52813720703125,
|
|
"logps/rejected": -392.0296936035156,
|
|
"logps/weighted_chosen": -2.7947998046875,
|
|
"logps/weighted_rejected": -3.697582960128784,
|
|
"loss": 0.6296,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -59.2109375,
|
|
"rewards/margins": 29.121875762939453,
|
|
"rewards/rejected": -88.32890319824219,
|
|
"rewards/weighted_accuracies": 0.653124988079071,
|
|
"rewards/weighted_chosen": -0.41761475801467896,
|
|
"rewards/weighted_margins": 0.38171082735061646,
|
|
"rewards/weighted_rejected": -0.7994705438613892,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.21460350693535724,
|
|
"grad_norm": 21.382999420166016,
|
|
"learning_rate": 9.611931925683266e-07,
|
|
"logits/chosen": -0.7703964114189148,
|
|
"logits/rejected": -0.808850109577179,
|
|
"logps/chosen": -367.3140563964844,
|
|
"logps/rejected": -348.0687561035156,
|
|
"logps/weighted_chosen": -2.711962938308716,
|
|
"logps/weighted_rejected": -3.4615721702575684,
|
|
"loss": 0.5758,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -61.02734375,
|
|
"rewards/margins": 23.316797256469727,
|
|
"rewards/rejected": -84.34687805175781,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.40519410371780396,
|
|
"rewards/weighted_margins": 0.521441638469696,
|
|
"rewards/weighted_rejected": -0.9261535406112671,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.21983773881182936,
|
|
"grad_norm": 23.030996322631836,
|
|
"learning_rate": 9.575867082172085e-07,
|
|
"logits/chosen": -0.7789466977119446,
|
|
"logits/rejected": -0.8260132074356079,
|
|
"logps/chosen": -372.22344970703125,
|
|
"logps/rejected": -367.0171813964844,
|
|
"logps/weighted_chosen": -3.114550828933716,
|
|
"logps/weighted_rejected": -3.364208936691284,
|
|
"loss": 0.6211,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -67.16816711425781,
|
|
"rewards/margins": 29.731639862060547,
|
|
"rewards/rejected": -96.90156555175781,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.5122925043106079,
|
|
"rewards/weighted_margins": 0.5218566656112671,
|
|
"rewards/weighted_rejected": -1.0339782238006592,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.22507197068830148,
|
|
"grad_norm": 16.442333221435547,
|
|
"learning_rate": 9.538273936226673e-07,
|
|
"logits/chosen": -0.830523669719696,
|
|
"logits/rejected": -0.8667358160018921,
|
|
"logps/chosen": -328.4546813964844,
|
|
"logps/rejected": -347.9593811035156,
|
|
"logps/weighted_chosen": -3.373584032058716,
|
|
"logps/weighted_rejected": -3.832958936691284,
|
|
"loss": 0.6425,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -62.920310974121094,
|
|
"rewards/margins": 20.668750762939453,
|
|
"rewards/rejected": -83.5894546508789,
|
|
"rewards/weighted_accuracies": 0.612500011920929,
|
|
"rewards/weighted_chosen": -0.459890753030777,
|
|
"rewards/weighted_margins": 0.39284056425094604,
|
|
"rewards/weighted_rejected": -0.8525451421737671,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.23030620256477363,
|
|
"grad_norm": 21.955875396728516,
|
|
"learning_rate": 9.499165043652391e-07,
|
|
"logits/chosen": -0.8598114252090454,
|
|
"logits/rejected": -0.868182361125946,
|
|
"logps/chosen": -358.21563720703125,
|
|
"logps/rejected": -356.26251220703125,
|
|
"logps/weighted_chosen": -3.4171142578125,
|
|
"logps/weighted_rejected": -3.6997313499450684,
|
|
"loss": 0.624,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -66.96504211425781,
|
|
"rewards/margins": 22.563282012939453,
|
|
"rewards/rejected": -89.5503921508789,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.5841079950332642,
|
|
"rewards/weighted_margins": 0.434326171875,
|
|
"rewards/weighted_rejected": -1.018707275390625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.23554043444124576,
|
|
"grad_norm": 75.56902313232422,
|
|
"learning_rate": 9.458553466501665e-07,
|
|
"logits/chosen": -0.9330536127090454,
|
|
"logits/rejected": -0.9642333984375,
|
|
"logps/chosen": -352.6187438964844,
|
|
"logps/rejected": -336.0218811035156,
|
|
"logps/weighted_chosen": -3.4129395484924316,
|
|
"logps/weighted_rejected": -3.74462890625,
|
|
"loss": 0.6566,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -66.412109375,
|
|
"rewards/margins": 25.757617950439453,
|
|
"rewards/rejected": -92.181640625,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.689007580280304,
|
|
"rewards/weighted_margins": 0.4539245665073395,
|
|
"rewards/weighted_rejected": -1.143212914466858,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.24077466631771788,
|
|
"grad_norm": 19.516427993774414,
|
|
"learning_rate": 9.416452768711366e-07,
|
|
"logits/chosen": -0.945111095905304,
|
|
"logits/rejected": -0.9787231683731079,
|
|
"logps/chosen": -369.3671875,
|
|
"logps/rejected": -358.9624938964844,
|
|
"logps/weighted_chosen": -3.1959471702575684,
|
|
"logps/weighted_rejected": -3.948193311691284,
|
|
"loss": 0.6392,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -77.17265319824219,
|
|
"rewards/margins": 21.916015625,
|
|
"rewards/rejected": -99.0796890258789,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.67822265625,
|
|
"rewards/weighted_margins": 0.532788097858429,
|
|
"rewards/weighted_rejected": -1.2112305164337158,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.24600889819419,
|
|
"grad_norm": 19.182979583740234,
|
|
"learning_rate": 9.372877011572557e-07,
|
|
"logits/chosen": -0.9224609136581421,
|
|
"logits/rejected": -0.9388214349746704,
|
|
"logps/chosen": -391.6937561035156,
|
|
"logps/rejected": -377.0625,
|
|
"logps/weighted_chosen": -3.224560499191284,
|
|
"logps/weighted_rejected": -3.783252000808716,
|
|
"loss": 0.6162,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -80.95976257324219,
|
|
"rewards/margins": 21.617578506469727,
|
|
"rewards/rejected": -102.59883117675781,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.684155285358429,
|
|
"rewards/weighted_margins": 0.5555480718612671,
|
|
"rewards/weighted_rejected": -1.23956298828125,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.2512431300706621,
|
|
"grad_norm": 31.75469970703125,
|
|
"learning_rate": 9.327840749034141e-07,
|
|
"logits/chosen": -0.969561755657196,
|
|
"logits/rejected": -0.998791515827179,
|
|
"logps/chosen": -362.1859436035156,
|
|
"logps/rejected": -385.29998779296875,
|
|
"logps/weighted_chosen": -3.0771241188049316,
|
|
"logps/weighted_rejected": -4.388257026672363,
|
|
"loss": 0.6296,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -75.1123046875,
|
|
"rewards/margins": 33.66425704956055,
|
|
"rewards/rejected": -108.75,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.606555163860321,
|
|
"rewards/weighted_margins": 0.612231433391571,
|
|
"rewards/weighted_rejected": -1.2182190418243408,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.2564773619471343,
|
|
"grad_norm": 37.024818420410156,
|
|
"learning_rate": 9.281359022841965e-07,
|
|
"logits/chosen": -0.846588134765625,
|
|
"logits/rejected": -0.859790027141571,
|
|
"logps/chosen": -352.46405029296875,
|
|
"logps/rejected": -355.24688720703125,
|
|
"logps/weighted_chosen": -3.219531297683716,
|
|
"logps/weighted_rejected": -4.648681640625,
|
|
"loss": 0.5897,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -77.3949203491211,
|
|
"rewards/margins": 32.93046951293945,
|
|
"rewards/rejected": -110.32890319824219,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -0.7288268804550171,
|
|
"rewards/weighted_margins": 0.6741577386856079,
|
|
"rewards/weighted_rejected": -1.403161644935608,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"grad_norm": 17.740766525268555,
|
|
"learning_rate": 9.233447357514989e-07,
|
|
"logits/chosen": -0.8205505609512329,
|
|
"logits/rejected": -0.863543689250946,
|
|
"logps/chosen": -375.52032470703125,
|
|
"logps/rejected": -378.3500061035156,
|
|
"logps/weighted_chosen": -3.53125,
|
|
"logps/weighted_rejected": -4.106689453125,
|
|
"loss": 0.6305,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -77.591796875,
|
|
"rewards/margins": 30.978906631469727,
|
|
"rewards/rejected": -108.54609680175781,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.6407378911972046,
|
|
"rewards/weighted_margins": 0.6662231683731079,
|
|
"rewards/weighted_rejected": -1.30645751953125,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"eval_logits/chosen": -0.9400458931922913,
|
|
"eval_logits/rejected": -0.955981433391571,
|
|
"eval_logps/chosen": -371.72900390625,
|
|
"eval_logps/rejected": -379.6419982910156,
|
|
"eval_logps/weighted_chosen": -3.214712381362915,
|
|
"eval_logps/weighted_rejected": -4.0158867835998535,
|
|
"eval_loss": 0.6316163539886475,
|
|
"eval_rewards/accuracies": 0.6349999904632568,
|
|
"eval_rewards/chosen": -82.98784637451172,
|
|
"eval_rewards/margins": 28.939437866210938,
|
|
"eval_rewards/rejected": -111.93875122070312,
|
|
"eval_rewards/weighted_accuracies": 0.6725000143051147,
|
|
"eval_rewards/weighted_chosen": -0.6669993996620178,
|
|
"eval_rewards/weighted_margins": 0.5506796836853027,
|
|
"eval_rewards/weighted_rejected": -1.2176789045333862,
|
|
"eval_runtime": 1162.5522,
|
|
"eval_samples_per_second": 1.72,
|
|
"eval_steps_per_second": 0.43,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.2669458257000785,
|
|
"grad_norm": 59.14344787597656,
|
|
"learning_rate": 9.184121755160232e-07,
|
|
"logits/chosen": -0.9093383550643921,
|
|
"logits/rejected": -0.9390915036201477,
|
|
"logps/chosen": -378.3890686035156,
|
|
"logps/rejected": -401.62811279296875,
|
|
"logps/weighted_chosen": -3.4715576171875,
|
|
"logps/weighted_rejected": -4.080712795257568,
|
|
"loss": 0.6505,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -77.12968444824219,
|
|
"rewards/margins": 35.939842224121094,
|
|
"rewards/rejected": -113.0718765258789,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.6525024175643921,
|
|
"rewards/weighted_margins": 0.52886962890625,
|
|
"rewards/weighted_rejected": -1.1813843250274658,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.2721800575765506,
|
|
"grad_norm": 33.64823913574219,
|
|
"learning_rate": 9.133398690128193e-07,
|
|
"logits/chosen": -0.942626953125,
|
|
"logits/rejected": -0.965716540813446,
|
|
"logps/chosen": -400.1890563964844,
|
|
"logps/rejected": -403.63592529296875,
|
|
"logps/weighted_chosen": -3.2196044921875,
|
|
"logps/weighted_rejected": -4.1656494140625,
|
|
"loss": 0.6062,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -78.64530944824219,
|
|
"rewards/margins": 44.30859375,
|
|
"rewards/rejected": -122.9917984008789,
|
|
"rewards/weighted_accuracies": 0.7406250238418579,
|
|
"rewards/weighted_chosen": -0.48695677518844604,
|
|
"rewards/weighted_margins": 0.6421569585800171,
|
|
"rewards/weighted_rejected": -1.129034399986267,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.27741428945302277,
|
|
"grad_norm": 36.97419738769531,
|
|
"learning_rate": 9.081295103510554e-07,
|
|
"logits/chosen": -0.9060531854629517,
|
|
"logits/rejected": -0.954547107219696,
|
|
"logps/chosen": -377.2640686035156,
|
|
"logps/rejected": -402.17498779296875,
|
|
"logps/weighted_chosen": -2.8717284202575684,
|
|
"logps/weighted_rejected": -4.353662014007568,
|
|
"loss": 0.5138,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -93.2210922241211,
|
|
"rewards/margins": 39.227149963378906,
|
|
"rewards/rejected": -132.44140625,
|
|
"rewards/weighted_accuracies": 0.7250000238418579,
|
|
"rewards/weighted_chosen": -0.5675109624862671,
|
|
"rewards/weighted_margins": 0.93292236328125,
|
|
"rewards/weighted_rejected": -1.501245141029358,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.2826485213294949,
|
|
"grad_norm": 23.371498107910156,
|
|
"learning_rate": 9.027828397481989e-07,
|
|
"logits/chosen": -0.8855453729629517,
|
|
"logits/rejected": -0.911120593547821,
|
|
"logps/chosen": -357.30078125,
|
|
"logps/rejected": -385.8531188964844,
|
|
"logps/weighted_chosen": -3.6192626953125,
|
|
"logps/weighted_rejected": -4.450634956359863,
|
|
"loss": 0.6284,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -96.7691421508789,
|
|
"rewards/margins": 33.722267150878906,
|
|
"rewards/rejected": -130.5457000732422,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": -0.788897693157196,
|
|
"rewards/weighted_margins": 0.5829833745956421,
|
|
"rewards/weighted_rejected": -1.3713562488555908,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.287882753205967,
|
|
"grad_norm": 21.5579833984375,
|
|
"learning_rate": 8.973016429487988e-07,
|
|
"logits/chosen": -0.9223998785018921,
|
|
"logits/rejected": -0.9320526123046875,
|
|
"logps/chosen": -376.43438720703125,
|
|
"logps/rejected": -393.18438720703125,
|
|
"logps/weighted_chosen": -3.503002882003784,
|
|
"logps/weighted_rejected": -3.9817872047424316,
|
|
"loss": 0.6093,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -94.5347671508789,
|
|
"rewards/margins": 44.30546951293945,
|
|
"rewards/rejected": -138.84414672851562,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.7487853765487671,
|
|
"rewards/weighted_margins": 0.62567138671875,
|
|
"rewards/weighted_rejected": -1.3746826648712158,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.29311698508243916,
|
|
"grad_norm": 55.23554992675781,
|
|
"learning_rate": 8.916877506280601e-07,
|
|
"logits/chosen": -0.969989001750946,
|
|
"logits/rejected": -0.9665802121162415,
|
|
"logps/chosen": -383.078125,
|
|
"logps/rejected": -397.75,
|
|
"logps/weighted_chosen": -3.5821290016174316,
|
|
"logps/weighted_rejected": -4.008593559265137,
|
|
"loss": 0.6282,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -102.19023132324219,
|
|
"rewards/margins": 41.46562576293945,
|
|
"rewards/rejected": -143.642578125,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.7835754156112671,
|
|
"rewards/weighted_margins": 0.644238293170929,
|
|
"rewards/weighted_rejected": -1.427978515625,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.29835121695891126,
|
|
"grad_norm": 18.939470291137695,
|
|
"learning_rate": 8.85943037780415e-07,
|
|
"logits/chosen": -1.0394058227539062,
|
|
"logits/rejected": -1.042639136314392,
|
|
"logps/chosen": -384.078125,
|
|
"logps/rejected": -366.9984436035156,
|
|
"logps/weighted_chosen": -3.3567872047424316,
|
|
"logps/weighted_rejected": -3.9019775390625,
|
|
"loss": 0.5908,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -97.7894515991211,
|
|
"rewards/margins": 27.4404296875,
|
|
"rewards/rejected": -125.2328109741211,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -0.7109375,
|
|
"rewards/weighted_margins": 0.566577136516571,
|
|
"rewards/weighted_rejected": -1.2770659923553467,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.3035854488353834,
|
|
"grad_norm": 54.97072982788086,
|
|
"learning_rate": 8.800694230932884e-07,
|
|
"logits/chosen": -0.958050549030304,
|
|
"logits/rejected": -0.9729766845703125,
|
|
"logps/chosen": -381.4140625,
|
|
"logps/rejected": -385.4296875,
|
|
"logps/weighted_chosen": -2.9930176734924316,
|
|
"logps/weighted_rejected": -3.6538329124450684,
|
|
"loss": 0.6283,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -91.8843765258789,
|
|
"rewards/margins": 28.43359375,
|
|
"rewards/rejected": -120.30078125,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.5365799069404602,
|
|
"rewards/weighted_margins": 0.4689392149448395,
|
|
"rewards/weighted_rejected": -1.005883812904358,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.30881968071185556,
|
|
"grad_norm": 187.63499450683594,
|
|
"learning_rate": 8.740688683062723e-07,
|
|
"logits/chosen": -1.0116729736328125,
|
|
"logits/rejected": -1.0273834466934204,
|
|
"logps/chosen": -427.6859436035156,
|
|
"logps/rejected": -400.0640563964844,
|
|
"logps/weighted_chosen": -2.997143507003784,
|
|
"logps/weighted_rejected": -3.7496094703674316,
|
|
"loss": 0.6363,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -98.6539077758789,
|
|
"rewards/margins": 25.619726181030273,
|
|
"rewards/rejected": -124.2671890258789,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.539746105670929,
|
|
"rewards/weighted_margins": 0.48472291231155396,
|
|
"rewards/weighted_rejected": -1.023950219154358,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.31405391258832765,
|
|
"grad_norm": 17.906530380249023,
|
|
"learning_rate": 8.679433775559215e-07,
|
|
"logits/chosen": -0.991503894329071,
|
|
"logits/rejected": -1.0299193859100342,
|
|
"logps/chosen": -428.88592529296875,
|
|
"logps/rejected": -425.1625061035156,
|
|
"logps/weighted_chosen": -2.877368211746216,
|
|
"logps/weighted_rejected": -3.985302686691284,
|
|
"loss": 0.5932,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -104.1353530883789,
|
|
"rewards/margins": 35.59746170043945,
|
|
"rewards/rejected": -139.7765655517578,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.6103607416152954,
|
|
"rewards/weighted_margins": 0.5564330816268921,
|
|
"rewards/weighted_rejected": -1.166479468345642,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.3192881444647998,
|
|
"grad_norm": 33.20716094970703,
|
|
"learning_rate": 8.616949967063871e-07,
|
|
"logits/chosen": -0.9755920171737671,
|
|
"logits/rejected": -1.0198791027069092,
|
|
"logps/chosen": -370.45001220703125,
|
|
"logps/rejected": -400.4906311035156,
|
|
"logps/weighted_chosen": -3.3270506858825684,
|
|
"logps/weighted_rejected": -3.9315428733825684,
|
|
"loss": 0.6822,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -102.12422180175781,
|
|
"rewards/margins": 37.60078048706055,
|
|
"rewards/rejected": -139.7734375,
|
|
"rewards/weighted_accuracies": 0.640625,
|
|
"rewards/weighted_chosen": -0.7267090082168579,
|
|
"rewards/weighted_margins": 0.43719482421875,
|
|
"rewards/weighted_rejected": -1.163793921470642,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.3245223763412719,
|
|
"grad_norm": 13.562949180603027,
|
|
"learning_rate": 8.553258126661154e-07,
|
|
"logits/chosen": -1.00177001953125,
|
|
"logits/rejected": -1.010014295578003,
|
|
"logps/chosen": -390.65234375,
|
|
"logps/rejected": -402.65155029296875,
|
|
"logps/weighted_chosen": -3.4473876953125,
|
|
"logps/weighted_rejected": -4.26806640625,
|
|
"loss": 0.6605,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -110.80390930175781,
|
|
"rewards/margins": 35.68242263793945,
|
|
"rewards/rejected": -146.5031280517578,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.7691070437431335,
|
|
"rewards/weighted_margins": 0.534271240234375,
|
|
"rewards/weighted_rejected": -1.3035888671875,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.32975660821774405,
|
|
"grad_norm": 29.253734588623047,
|
|
"learning_rate": 8.488379526908368e-07,
|
|
"logits/chosen": -0.978869616985321,
|
|
"logits/rejected": -0.9867492914199829,
|
|
"logps/chosen": -406.9437561035156,
|
|
"logps/rejected": -431.9906311035156,
|
|
"logps/weighted_chosen": -3.346386671066284,
|
|
"logps/weighted_rejected": -4.080664157867432,
|
|
"loss": 0.5991,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -116.3648452758789,
|
|
"rewards/margins": 46.68730545043945,
|
|
"rewards/rejected": -163.06405639648438,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.757556140422821,
|
|
"rewards/weighted_margins": 0.597582995891571,
|
|
"rewards/weighted_rejected": -1.3551514148712158,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.33499084009421615,
|
|
"grad_norm": 28.59862518310547,
|
|
"learning_rate": 8.422335836730802e-07,
|
|
"logits/chosen": -0.983142077922821,
|
|
"logits/rejected": -0.9791107177734375,
|
|
"logps/chosen": -378.4984436035156,
|
|
"logps/rejected": -432.71563720703125,
|
|
"logps/weighted_chosen": -3.094311475753784,
|
|
"logps/weighted_rejected": -3.8768067359924316,
|
|
"loss": 0.6061,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -104.8597640991211,
|
|
"rewards/margins": 50.2001953125,
|
|
"rewards/rejected": -155.0695343017578,
|
|
"rewards/weighted_accuracies": 0.699999988079071,
|
|
"rewards/weighted_chosen": -0.6736419796943665,
|
|
"rewards/weighted_margins": 0.5768188238143921,
|
|
"rewards/weighted_rejected": -1.250451683998108,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.3402250719706883,
|
|
"grad_norm": 20.690876007080078,
|
|
"learning_rate": 8.355149114184485e-07,
|
|
"logits/chosen": -1.030615210533142,
|
|
"logits/rejected": -1.0146636962890625,
|
|
"logps/chosen": -416.39373779296875,
|
|
"logps/rejected": -445.0625,
|
|
"logps/weighted_chosen": -3.255859375,
|
|
"logps/weighted_rejected": -3.76806640625,
|
|
"loss": 0.6048,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -108.90547180175781,
|
|
"rewards/margins": 53.869140625,
|
|
"rewards/rejected": -162.74258422851562,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.700115978717804,
|
|
"rewards/weighted_margins": 0.5948852300643921,
|
|
"rewards/weighted_rejected": -1.2950623035430908,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.34545930384716045,
|
|
"grad_norm": 33.73557662963867,
|
|
"learning_rate": 8.286841799088963e-07,
|
|
"logits/chosen": -1.0694351196289062,
|
|
"logits/rejected": -1.0623047351837158,
|
|
"logps/chosen": -395.19219970703125,
|
|
"logps/rejected": -412.45001220703125,
|
|
"logps/weighted_chosen": -2.8808836936950684,
|
|
"logps/weighted_rejected": -3.636523485183716,
|
|
"loss": 0.6207,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -111.27070617675781,
|
|
"rewards/margins": 38.04765701293945,
|
|
"rewards/rejected": -149.3312530517578,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.649340808391571,
|
|
"rewards/weighted_margins": 0.5112365484237671,
|
|
"rewards/weighted_rejected": -1.1605103015899658,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.35069353572363254,
|
|
"grad_norm": 18.913761138916016,
|
|
"learning_rate": 8.217436705532599e-07,
|
|
"logits/chosen": -1.0736572742462158,
|
|
"logits/rejected": -1.090576171875,
|
|
"logps/chosen": -431.78125,
|
|
"logps/rejected": -421.4312438964844,
|
|
"logps/weighted_chosen": -3.0259766578674316,
|
|
"logps/weighted_rejected": -3.81298828125,
|
|
"loss": 0.6004,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -129.1144561767578,
|
|
"rewards/margins": 28.649608612060547,
|
|
"rewards/rejected": -157.74844360351562,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.6614929437637329,
|
|
"rewards/weighted_margins": 0.636364758014679,
|
|
"rewards/weighted_rejected": -1.2976195812225342,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.3559277676001047,
|
|
"grad_norm": 22.967056274414062,
|
|
"learning_rate": 8.14695701425284e-07,
|
|
"logits/chosen": -1.051629662513733,
|
|
"logits/rejected": -1.0860717296600342,
|
|
"logps/chosen": -432.98126220703125,
|
|
"logps/rejected": -426.5953063964844,
|
|
"logps/weighted_chosen": -3.1195311546325684,
|
|
"logps/weighted_rejected": -3.96337890625,
|
|
"loss": 0.5827,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -127.5308609008789,
|
|
"rewards/margins": 36.908592224121094,
|
|
"rewards/rejected": -164.46484375,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.686236560344696,
|
|
"rewards/weighted_margins": 0.6052306890487671,
|
|
"rewards/weighted_rejected": -1.2908813953399658,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.3611619994765768,
|
|
"grad_norm": 15.409049034118652,
|
|
"learning_rate": 8.075426264894046e-07,
|
|
"logits/chosen": -1.006170630455017,
|
|
"logits/rejected": -1.0271179676055908,
|
|
"logps/chosen": -434.3828125,
|
|
"logps/rejected": -452.90625,
|
|
"logps/weighted_chosen": -3.187304735183716,
|
|
"logps/weighted_rejected": -4.466699123382568,
|
|
"loss": 0.5501,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -128.9619140625,
|
|
"rewards/margins": 46.51679611206055,
|
|
"rewards/rejected": -175.45468139648438,
|
|
"rewards/weighted_accuracies": 0.737500011920929,
|
|
"rewards/weighted_chosen": -0.6887573003768921,
|
|
"rewards/weighted_margins": 0.76141357421875,
|
|
"rewards/weighted_rejected": -1.4498474597930908,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.36639623135304894,
|
|
"grad_norm": 23.13039207458496,
|
|
"learning_rate": 8.002868348145435e-07,
|
|
"logits/chosen": -0.9920509457588196,
|
|
"logits/rejected": -0.996777355670929,
|
|
"logps/chosen": -439.54998779296875,
|
|
"logps/rejected": -444.30780029296875,
|
|
"logps/weighted_chosen": -3.31005859375,
|
|
"logps/weighted_rejected": -3.691967725753784,
|
|
"loss": 0.6102,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -139.38632202148438,
|
|
"rewards/margins": 36.769142150878906,
|
|
"rewards/rejected": -176.1867218017578,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.8051727414131165,
|
|
"rewards/weighted_margins": 0.5639587640762329,
|
|
"rewards/weighted_rejected": -1.3695800304412842,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.3716304632295211,
|
|
"grad_norm": 38.38508224487305,
|
|
"learning_rate": 7.92930749776179e-07,
|
|
"logits/chosen": -1.0172607898712158,
|
|
"logits/rejected": -1.042083740234375,
|
|
"logps/chosen": -422.10546875,
|
|
"logps/rejected": -444.6812438964844,
|
|
"logps/weighted_chosen": -3.483569383621216,
|
|
"logps/weighted_rejected": -4.216650485992432,
|
|
"loss": 0.6316,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -150.42578125,
|
|
"rewards/margins": 33.904685974121094,
|
|
"rewards/rejected": -184.33438110351562,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.72515869140625,
|
|
"rewards/weighted_margins": 0.590161144733429,
|
|
"rewards/weighted_rejected": -1.3153502941131592,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.3768646951059932,
|
|
"grad_norm": 20.4711971282959,
|
|
"learning_rate": 7.854768282469582e-07,
|
|
"logits/chosen": -1.0770995616912842,
|
|
"logits/rejected": -1.114935278892517,
|
|
"logps/chosen": -401.88592529296875,
|
|
"logps/rejected": -458.3374938964844,
|
|
"logps/weighted_chosen": -3.099194288253784,
|
|
"logps/weighted_rejected": -3.990966796875,
|
|
"loss": 0.5978,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -124.43046569824219,
|
|
"rewards/margins": 56.217384338378906,
|
|
"rewards/rejected": -180.6015625,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.6303802728652954,
|
|
"rewards/weighted_margins": 0.679125964641571,
|
|
"rewards/weighted_rejected": -1.3104156255722046,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.38209892698246534,
|
|
"grad_norm": 15.2982177734375,
|
|
"learning_rate": 7.779275597761215e-07,
|
|
"logits/chosen": -1.0406615734100342,
|
|
"logits/rejected": -1.097131371498108,
|
|
"logps/chosen": -414.5625,
|
|
"logps/rejected": -462.5453186035156,
|
|
"logps/weighted_chosen": -3.3689942359924316,
|
|
"logps/weighted_rejected": -4.090185642242432,
|
|
"loss": 0.5689,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -138.33358764648438,
|
|
"rewards/margins": 55.842185974121094,
|
|
"rewards/rejected": -194.16171264648438,
|
|
"rewards/weighted_accuracies": 0.734375,
|
|
"rewards/weighted_chosen": -0.6730102300643921,
|
|
"rewards/weighted_margins": 0.699902355670929,
|
|
"rewards/weighted_rejected": -1.373620629310608,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.38733315885893743,
|
|
"grad_norm": 37.01581954956055,
|
|
"learning_rate": 7.702854657580126e-07,
|
|
"logits/chosen": -1.1022522449493408,
|
|
"logits/rejected": -1.1134154796600342,
|
|
"logps/chosen": -459.6156311035156,
|
|
"logps/rejected": -462.4125061035156,
|
|
"logps/weighted_chosen": -3.382946729660034,
|
|
"logps/weighted_rejected": -4.401709079742432,
|
|
"loss": 0.6148,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -164.8722686767578,
|
|
"rewards/margins": 43.28515625,
|
|
"rewards/rejected": -208.21133422851562,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.8088958859443665,
|
|
"rewards/weighted_margins": 0.698986828327179,
|
|
"rewards/weighted_rejected": -1.5074951648712158,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.3925673907354096,
|
|
"grad_norm": 26.924480438232422,
|
|
"learning_rate": 7.625530985899547e-07,
|
|
"logits/chosen": -1.0611861944198608,
|
|
"logits/rejected": -1.075714111328125,
|
|
"logps/chosen": -430.6312561035156,
|
|
"logps/rejected": -461.9828186035156,
|
|
"logps/weighted_chosen": -3.210217237472534,
|
|
"logps/weighted_rejected": -4.483691215515137,
|
|
"loss": 0.5815,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -163.78555297851562,
|
|
"rewards/margins": 53.288673400878906,
|
|
"rewards/rejected": -216.96054077148438,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.876666247844696,
|
|
"rewards/weighted_margins": 0.7388671636581421,
|
|
"rewards/weighted_rejected": -1.6160767078399658,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.39780162261188173,
|
|
"grad_norm": 18.056201934814453,
|
|
"learning_rate": 7.547330408197694e-07,
|
|
"logits/chosen": -1.0437713861465454,
|
|
"logits/rejected": -1.075250267982483,
|
|
"logps/chosen": -460.1875,
|
|
"logps/rejected": -450.8109436035156,
|
|
"logps/weighted_chosen": -3.229296922683716,
|
|
"logps/weighted_rejected": -4.191064357757568,
|
|
"loss": 0.6146,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -164.56640625,
|
|
"rewards/margins": 29.973241806030273,
|
|
"rewards/rejected": -194.6171875,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.7951415777206421,
|
|
"rewards/weighted_margins": 0.599578857421875,
|
|
"rewards/weighted_rejected": -1.395105004310608,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.40303585448835383,
|
|
"grad_norm": 16.393836975097656,
|
|
"learning_rate": 7.468279042832271e-07,
|
|
"logits/chosen": -1.0454833507537842,
|
|
"logits/rejected": -1.0705687999725342,
|
|
"logps/chosen": -416.57421875,
|
|
"logps/rejected": -483.1390686035156,
|
|
"logps/weighted_chosen": -3.134448289871216,
|
|
"logps/weighted_rejected": -3.8697752952575684,
|
|
"loss": 0.6132,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -136.8953094482422,
|
|
"rewards/margins": 56.886329650878906,
|
|
"rewards/rejected": -193.72305297851562,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.7508605718612671,
|
|
"rewards/weighted_margins": 0.5269104242324829,
|
|
"rewards/weighted_rejected": -1.277490258216858,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.408270086364826,
|
|
"grad_norm": 16.254776000976562,
|
|
"learning_rate": 7.388403292317154e-07,
|
|
"logits/chosen": -1.0213134288787842,
|
|
"logits/rejected": -1.072851538658142,
|
|
"logps/chosen": -445.00469970703125,
|
|
"logps/rejected": -451.1468811035156,
|
|
"logps/weighted_chosen": -3.063525438308716,
|
|
"logps/weighted_rejected": -3.897021532058716,
|
|
"loss": 0.6134,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -144.0207061767578,
|
|
"rewards/margins": 41.165626525878906,
|
|
"rewards/rejected": -185.15585327148438,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.752685546875,
|
|
"rewards/weighted_margins": 0.5895751714706421,
|
|
"rewards/weighted_rejected": -1.341894507408142,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.4135043182412981,
|
|
"grad_norm": 19.472450256347656,
|
|
"learning_rate": 7.307729834504154e-07,
|
|
"logits/chosen": -1.052435278892517,
|
|
"logits/rejected": -1.10076904296875,
|
|
"logps/chosen": -457.2640686035156,
|
|
"logps/rejected": -488.16876220703125,
|
|
"logps/weighted_chosen": -3.143115282058716,
|
|
"logps/weighted_rejected": -4.254638671875,
|
|
"loss": 0.6137,
|
|
"rewards/accuracies": 0.5843750238418579,
|
|
"rewards/chosen": -175.60116577148438,
|
|
"rewards/margins": 46.61640548706055,
|
|
"rewards/rejected": -222.21328735351562,
|
|
"rewards/weighted_accuracies": 0.6812499761581421,
|
|
"rewards/weighted_chosen": -0.878021240234375,
|
|
"rewards/weighted_margins": 0.650500476360321,
|
|
"rewards/weighted_rejected": -1.5286986827850342,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.4187385501177702,
|
|
"grad_norm": 24.911523818969727,
|
|
"learning_rate": 7.226285613672847e-07,
|
|
"logits/chosen": -1.0021483898162842,
|
|
"logits/rejected": -1.031951904296875,
|
|
"logps/chosen": -466.4765625,
|
|
"logps/rejected": -547.578125,
|
|
"logps/weighted_chosen": -3.3023438453674316,
|
|
"logps/weighted_rejected": -4.483691215515137,
|
|
"loss": 0.6142,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -189.1164093017578,
|
|
"rewards/margins": 79.8050765991211,
|
|
"rewards/rejected": -268.85467529296875,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.978710949420929,
|
|
"rewards/weighted_margins": 0.6761840581893921,
|
|
"rewards/weighted_rejected": -1.6549804210662842,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.4239727819942423,
|
|
"grad_norm": 16.060869216918945,
|
|
"learning_rate": 7.144097831531398e-07,
|
|
"logits/chosen": -0.978619396686554,
|
|
"logits/rejected": -1.0038635730743408,
|
|
"logps/chosen": -456.95623779296875,
|
|
"logps/rejected": -505.2093811035156,
|
|
"logps/weighted_chosen": -3.171826124191284,
|
|
"logps/weighted_rejected": -4.205761909484863,
|
|
"loss": 0.5645,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -186.97421264648438,
|
|
"rewards/margins": 54.0078125,
|
|
"rewards/rejected": -240.85311889648438,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.788342297077179,
|
|
"rewards/weighted_margins": 0.7865844964981079,
|
|
"rewards/weighted_rejected": -1.574896216392517,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.42920701387071447,
|
|
"grad_norm": 23.292619705200195,
|
|
"learning_rate": 7.061193938131396e-07,
|
|
"logits/chosen": -0.9266418218612671,
|
|
"logits/rejected": -0.9759277105331421,
|
|
"logps/chosen": -492.421875,
|
|
"logps/rejected": -497.2515563964844,
|
|
"logps/weighted_chosen": -3.4576172828674316,
|
|
"logps/weighted_rejected": -4.116650581359863,
|
|
"loss": 0.5666,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -192.64022827148438,
|
|
"rewards/margins": 39.222267150878906,
|
|
"rewards/rejected": -231.8562469482422,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.797924816608429,
|
|
"rewards/weighted_margins": 0.775280773639679,
|
|
"rewards/weighted_rejected": -1.573089599609375,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.4344412457471866,
|
|
"grad_norm": 23.763275146484375,
|
|
"learning_rate": 6.977601622699789e-07,
|
|
"logits/chosen": -0.9908691644668579,
|
|
"logits/rejected": -1.057653784751892,
|
|
"logps/chosen": -459.80157470703125,
|
|
"logps/rejected": -544.8687744140625,
|
|
"logps/weighted_chosen": -3.350512742996216,
|
|
"logps/weighted_rejected": -4.385839939117432,
|
|
"loss": 0.5077,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -175.025390625,
|
|
"rewards/margins": 90.8042984008789,
|
|
"rewards/rejected": -265.86798095703125,
|
|
"rewards/weighted_accuracies": 0.7281249761581421,
|
|
"rewards/weighted_chosen": -0.699688732624054,
|
|
"rewards/weighted_margins": 0.9197998046875,
|
|
"rewards/weighted_rejected": -1.61993408203125,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.4396754776236587,
|
|
"grad_norm": 29.07372283935547,
|
|
"learning_rate": 6.893348804390882e-07,
|
|
"logits/chosen": -1.094964623451233,
|
|
"logits/rejected": -1.1045074462890625,
|
|
"logps/chosen": -521.2327880859375,
|
|
"logps/rejected": -545.9468994140625,
|
|
"logps/weighted_chosen": -3.6615967750549316,
|
|
"logps/weighted_rejected": -4.320361137390137,
|
|
"loss": 0.5747,
|
|
"rewards/accuracies": 0.5843750238418579,
|
|
"rewards/chosen": -225.86563110351562,
|
|
"rewards/margins": 60.388671875,
|
|
"rewards/rejected": -286.2398376464844,
|
|
"rewards/weighted_accuracies": 0.715624988079071,
|
|
"rewards/weighted_chosen": -0.912017822265625,
|
|
"rewards/weighted_margins": 0.748242199420929,
|
|
"rewards/weighted_rejected": -1.660058617591858,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.44490970950013087,
|
|
"grad_norm": 23.878381729125977,
|
|
"learning_rate": 6.808463622961578e-07,
|
|
"logits/chosen": -1.0891234874725342,
|
|
"logits/rejected": -1.1309936046600342,
|
|
"logps/chosen": -545.3046875,
|
|
"logps/rejected": -615.7484130859375,
|
|
"logps/weighted_chosen": -3.533984422683716,
|
|
"logps/weighted_rejected": -4.504004001617432,
|
|
"loss": 0.5472,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -252.8015594482422,
|
|
"rewards/margins": 81.0531234741211,
|
|
"rewards/rejected": -333.59295654296875,
|
|
"rewards/weighted_accuracies": 0.715624988079071,
|
|
"rewards/weighted_chosen": -0.967456042766571,
|
|
"rewards/weighted_margins": 0.86865234375,
|
|
"rewards/weighted_rejected": -1.835351586341858,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.45014394137660296,
|
|
"grad_norm": 101.535888671875,
|
|
"learning_rate": 6.722974429372925e-07,
|
|
"logits/chosen": -1.0688354969024658,
|
|
"logits/rejected": -1.1046874523162842,
|
|
"logps/chosen": -578.9781494140625,
|
|
"logps/rejected": -604.3499755859375,
|
|
"logps/weighted_chosen": -3.37939453125,
|
|
"logps/weighted_rejected": -5.040380954742432,
|
|
"loss": 0.5051,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -275.2124938964844,
|
|
"rewards/margins": 66.9710922241211,
|
|
"rewards/rejected": -342.20001220703125,
|
|
"rewards/weighted_accuracies": 0.7749999761581421,
|
|
"rewards/weighted_chosen": -1.086279273033142,
|
|
"rewards/weighted_margins": 1.05950927734375,
|
|
"rewards/weighted_rejected": -2.1461181640625,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.4553781732530751,
|
|
"grad_norm": 66.56680297851562,
|
|
"learning_rate": 6.636909776321128e-07,
|
|
"logits/chosen": -1.1214478015899658,
|
|
"logits/rejected": -1.11016845703125,
|
|
"logps/chosen": -493.3843688964844,
|
|
"logps/rejected": -590.8125,
|
|
"logps/weighted_chosen": -3.670654296875,
|
|
"logps/weighted_rejected": -4.723730564117432,
|
|
"loss": 0.5107,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -231.6789093017578,
|
|
"rewards/margins": 94.42109680175781,
|
|
"rewards/rejected": -326.2250061035156,
|
|
"rewards/weighted_accuracies": 0.746874988079071,
|
|
"rewards/weighted_chosen": -1.0464904308319092,
|
|
"rewards/weighted_margins": 0.9440551996231079,
|
|
"rewards/weighted_rejected": -1.99102783203125,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.46061240512954726,
|
|
"grad_norm": 23.494997024536133,
|
|
"learning_rate": 6.550298408701174e-07,
|
|
"logits/chosen": -1.094885230064392,
|
|
"logits/rejected": -1.1415894031524658,
|
|
"logps/chosen": -534.1663818359375,
|
|
"logps/rejected": -621.9547119140625,
|
|
"logps/weighted_chosen": -3.794140577316284,
|
|
"logps/weighted_rejected": -5.148291110992432,
|
|
"loss": 0.5174,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -247.2595672607422,
|
|
"rewards/margins": 92.109375,
|
|
"rewards/rejected": -339.3968811035156,
|
|
"rewards/weighted_accuracies": 0.753125011920929,
|
|
"rewards/weighted_chosen": -0.9449707269668579,
|
|
"rewards/weighted_margins": 0.938586413860321,
|
|
"rewards/weighted_rejected": -1.8829224109649658,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.46584663700601936,
|
|
"grad_norm": 27.359371185302734,
|
|
"learning_rate": 6.463169254006276e-07,
|
|
"logits/chosen": -1.1160705089569092,
|
|
"logits/rejected": -1.157629370689392,
|
|
"logps/chosen": -538.1109619140625,
|
|
"logps/rejected": -562.7062377929688,
|
|
"logps/weighted_chosen": -3.73779296875,
|
|
"logps/weighted_rejected": -4.817724704742432,
|
|
"loss": 0.5154,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -256.6499938964844,
|
|
"rewards/margins": 59.66523361206055,
|
|
"rewards/rejected": -316.37225341796875,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -1.019891381263733,
|
|
"rewards/weighted_margins": 1.008874535560608,
|
|
"rewards/weighted_rejected": -2.028430223464966,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.4710808688824915,
|
|
"grad_norm": 20.988025665283203,
|
|
"learning_rate": 6.375551412666326e-07,
|
|
"logits/chosen": -1.0879943370819092,
|
|
"logits/rejected": -1.1199951171875,
|
|
"logps/chosen": -514.9187622070312,
|
|
"logps/rejected": -545.7906494140625,
|
|
"logps/weighted_chosen": -3.4129395484924316,
|
|
"logps/weighted_rejected": -4.592138767242432,
|
|
"loss": 0.6113,
|
|
"rewards/accuracies": 0.5843750238418579,
|
|
"rewards/chosen": -240.97891235351562,
|
|
"rewards/margins": 48.454689025878906,
|
|
"rewards/rejected": -289.4117126464844,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -1.1000854969024658,
|
|
"rewards/weighted_margins": 0.7867187261581421,
|
|
"rewards/weighted_rejected": -1.887121558189392,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.4763151007589636,
|
|
"grad_norm": 29.907148361206055,
|
|
"learning_rate": 6.287474148328583e-07,
|
|
"logits/chosen": -1.0193603038787842,
|
|
"logits/rejected": -0.9993133544921875,
|
|
"logps/chosen": -474.3374938964844,
|
|
"logps/rejected": -501.23748779296875,
|
|
"logps/weighted_chosen": -3.7416014671325684,
|
|
"logps/weighted_rejected": -5.238329887390137,
|
|
"loss": 0.5727,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -203.36563110351562,
|
|
"rewards/margins": 50.95781326293945,
|
|
"rewards/rejected": -254.2734375,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -1.043182373046875,
|
|
"rewards/weighted_margins": 0.8107665777206421,
|
|
"rewards/weighted_rejected": -1.8539307117462158,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.48154933263543576,
|
|
"grad_norm": 36.32797622680664,
|
|
"learning_rate": 6.198966878083857e-07,
|
|
"logits/chosen": -1.0350799560546875,
|
|
"logits/rejected": -1.0523681640625,
|
|
"logps/chosen": -488.9765625,
|
|
"logps/rejected": -553.484375,
|
|
"logps/weighted_chosen": -3.7232666015625,
|
|
"logps/weighted_rejected": -4.598974704742432,
|
|
"loss": 0.5581,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -217.74844360351562,
|
|
"rewards/margins": 68.5667953491211,
|
|
"rewards/rejected": -286.3515625,
|
|
"rewards/weighted_accuracies": 0.734375,
|
|
"rewards/weighted_chosen": -1.01214599609375,
|
|
"rewards/weighted_margins": 0.775787353515625,
|
|
"rewards/weighted_rejected": -1.787988305091858,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.48678356451190785,
|
|
"grad_norm": 40.16273880004883,
|
|
"learning_rate": 6.110059162641439e-07,
|
|
"logits/chosen": -1.0597412586212158,
|
|
"logits/rejected": -1.0781066417694092,
|
|
"logps/chosen": -513.3226318359375,
|
|
"logps/rejected": -559.4593505859375,
|
|
"logps/weighted_chosen": -3.147705078125,
|
|
"logps/weighted_rejected": -4.214404106140137,
|
|
"loss": 0.5481,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -231.1085968017578,
|
|
"rewards/margins": 61.108985900878906,
|
|
"rewards/rejected": -292.21954345703125,
|
|
"rewards/weighted_accuracies": 0.721875011920929,
|
|
"rewards/weighted_chosen": -0.8689819574356079,
|
|
"rewards/weighted_margins": 0.7522827386856079,
|
|
"rewards/weighted_rejected": -1.6212646961212158,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.49201779638838,
|
|
"grad_norm": 29.65454864501953,
|
|
"learning_rate": 6.020780696456059e-07,
|
|
"logits/chosen": -1.072198510169983,
|
|
"logits/rejected": -1.104650855064392,
|
|
"logps/chosen": -511.18280029296875,
|
|
"logps/rejected": -601.0250244140625,
|
|
"logps/weighted_chosen": -3.1954102516174316,
|
|
"logps/weighted_rejected": -4.517724514007568,
|
|
"loss": 0.5407,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -240.52108764648438,
|
|
"rewards/margins": 93.419921875,
|
|
"rewards/rejected": -333.953125,
|
|
"rewards/weighted_accuracies": 0.731249988079071,
|
|
"rewards/weighted_chosen": -1.0927855968475342,
|
|
"rewards/weighted_margins": 0.8496459722518921,
|
|
"rewards/weighted_rejected": -1.942968726158142,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.49725202826485215,
|
|
"grad_norm": 210.50332641601562,
|
|
"learning_rate": 5.931161297810185e-07,
|
|
"logits/chosen": -1.132635474205017,
|
|
"logits/rejected": -1.1451904773712158,
|
|
"logps/chosen": -574.6031494140625,
|
|
"logps/rejected": -629.4656372070312,
|
|
"logps/weighted_chosen": -4.126172065734863,
|
|
"logps/weighted_rejected": -5.016747951507568,
|
|
"loss": 0.5998,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": -301.52264404296875,
|
|
"rewards/margins": 66.5218734741211,
|
|
"rewards/rejected": -368.080078125,
|
|
"rewards/weighted_accuracies": 0.6656249761581421,
|
|
"rewards/weighted_chosen": -1.307519555091858,
|
|
"rewards/weighted_margins": 0.74249267578125,
|
|
"rewards/weighted_rejected": -2.0491180419921875,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.5024862601413242,
|
|
"grad_norm": 113.20726013183594,
|
|
"learning_rate": 5.841230898854959e-07,
|
|
"logits/chosen": -1.070550560951233,
|
|
"logits/rejected": -1.0872802734375,
|
|
"logps/chosen": -652.0281372070312,
|
|
"logps/rejected": -711.1765747070312,
|
|
"logps/weighted_chosen": -3.9981932640075684,
|
|
"logps/weighted_rejected": -5.147070407867432,
|
|
"loss": 0.6329,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -351.828125,
|
|
"rewards/margins": 97.4195327758789,
|
|
"rewards/rejected": -449.2261657714844,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -1.564361572265625,
|
|
"rewards/weighted_margins": 0.872241199016571,
|
|
"rewards/weighted_rejected": -2.436718702316284,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.5077204920177963,
|
|
"grad_norm": 22.200820922851562,
|
|
"learning_rate": 5.751019535613102e-07,
|
|
"logits/chosen": -0.987274169921875,
|
|
"logits/rejected": -1.0052611827850342,
|
|
"logps/chosen": -531.3624877929688,
|
|
"logps/rejected": -610.1218872070312,
|
|
"logps/weighted_chosen": -3.8460450172424316,
|
|
"logps/weighted_rejected": -5.256982326507568,
|
|
"loss": 0.5675,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -271.6910095214844,
|
|
"rewards/margins": 91.53633117675781,
|
|
"rewards/rejected": -363.22344970703125,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -1.357843041419983,
|
|
"rewards/weighted_margins": 1.01031494140625,
|
|
"rewards/weighted_rejected": -2.367443799972534,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.5129547238942685,
|
|
"grad_norm": 17.362323760986328,
|
|
"learning_rate": 5.660557337947117e-07,
|
|
"logits/chosen": -0.9707549810409546,
|
|
"logits/rejected": -0.983325183391571,
|
|
"logps/chosen": -549.1953125,
|
|
"logps/rejected": -586.7874755859375,
|
|
"logps/weighted_chosen": -3.279223680496216,
|
|
"logps/weighted_rejected": -4.507519721984863,
|
|
"loss": 0.5466,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -254.6687469482422,
|
|
"rewards/margins": 74.56758117675781,
|
|
"rewards/rejected": -329.2632751464844,
|
|
"rewards/weighted_accuracies": 0.699999988079071,
|
|
"rewards/weighted_chosen": -1.0866820812225342,
|
|
"rewards/weighted_margins": 0.8655151128768921,
|
|
"rewards/weighted_rejected": -1.9528076648712158,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.5181889557707406,
|
|
"grad_norm": 39.01738739013672,
|
|
"learning_rate": 5.569874519496174e-07,
|
|
"logits/chosen": -0.963134765625,
|
|
"logits/rejected": -1.01763916015625,
|
|
"logps/chosen": -488.7406311035156,
|
|
"logps/rejected": -554.3687744140625,
|
|
"logps/weighted_chosen": -3.665576219558716,
|
|
"logps/weighted_rejected": -4.876318454742432,
|
|
"loss": 0.5929,
|
|
"rewards/accuracies": 0.590624988079071,
|
|
"rewards/chosen": -213.4460906982422,
|
|
"rewards/margins": 73.8890609741211,
|
|
"rewards/rejected": -287.3500061035156,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.959338366985321,
|
|
"rewards/weighted_margins": 0.7813965082168579,
|
|
"rewards/weighted_rejected": -1.740045189857483,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"grad_norm": 33.2608642578125,
|
|
"learning_rate": 5.47900136758499e-07,
|
|
"logits/chosen": -0.9298340082168579,
|
|
"logits/rejected": -0.989898681640625,
|
|
"logps/chosen": -527.16015625,
|
|
"logps/rejected": -566.2453002929688,
|
|
"logps/weighted_chosen": -3.71044921875,
|
|
"logps/weighted_rejected": -4.887304782867432,
|
|
"loss": 0.5395,
|
|
"rewards/accuracies": 0.590624988079071,
|
|
"rewards/chosen": -259.56170654296875,
|
|
"rewards/margins": 64.66015625,
|
|
"rewards/rejected": -324.302734375,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -1.119836449623108,
|
|
"rewards/weighted_margins": 0.944445788860321,
|
|
"rewards/weighted_rejected": -2.063854932785034,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"eval_logits/chosen": -1.0472733974456787,
|
|
"eval_logits/rejected": -1.0595996379852295,
|
|
"eval_logps/chosen": -573.6119995117188,
|
|
"eval_logps/rejected": -629.1840209960938,
|
|
"eval_logps/weighted_chosen": -3.815713405609131,
|
|
"eval_logps/weighted_rejected": -4.930161476135254,
|
|
"eval_loss": 0.5728335976600647,
|
|
"eval_rewards/accuracies": 0.5989999771118164,
|
|
"eval_rewards/chosen": -284.8971252441406,
|
|
"eval_rewards/margins": 76.59700012207031,
|
|
"eval_rewards/rejected": -361.5224914550781,
|
|
"eval_rewards/weighted_accuracies": 0.7070000171661377,
|
|
"eval_rewards/weighted_chosen": -1.2679998874664307,
|
|
"eval_rewards/weighted_margins": 0.8639541268348694,
|
|
"eval_rewards/weighted_rejected": -2.1319541931152344,
|
|
"eval_runtime": 1366.4223,
|
|
"eval_samples_per_second": 1.464,
|
|
"eval_steps_per_second": 0.366,
|
|
"step": 1000
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1911,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|