2200 lines
86 KiB
JSON
2200 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5234231876472127,
|
|
"eval_steps": 500,
|
|
"global_step": 1000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0005234231876472127,
|
|
"grad_norm": 136.4956817626953,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.40118408203125,
|
|
"logits/rejected": -0.41802978515625,
|
|
"logps/chosen": -297.609375,
|
|
"logps/rejected": -247.84375,
|
|
"logps/weighted_chosen": -4.50634765625,
|
|
"logps/weighted_rejected": -3.43408203125,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"rewards/weighted_accuracies": 0.0,
|
|
"rewards/weighted_chosen": 0.0,
|
|
"rewards/weighted_margins": 0.0,
|
|
"rewards/weighted_rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.005234231876472127,
|
|
"grad_norm": 269.8279113769531,
|
|
"learning_rate": 4.6875e-08,
|
|
"logits/chosen": -0.3176456093788147,
|
|
"logits/rejected": -0.3530849814414978,
|
|
"logps/chosen": -275.5694580078125,
|
|
"logps/rejected": -255.875,
|
|
"logps/weighted_chosen": -2.478325843811035,
|
|
"logps/weighted_rejected": -2.635009765625,
|
|
"loss": 0.6911,
|
|
"rewards/accuracies": 0.2604166567325592,
|
|
"rewards/chosen": -0.0588107630610466,
|
|
"rewards/margins": -0.0334201380610466,
|
|
"rewards/rejected": -0.025390625,
|
|
"rewards/weighted_accuracies": 0.3229166567325592,
|
|
"rewards/weighted_chosen": 0.0010899438057094812,
|
|
"rewards/weighted_margins": 0.0014837053604424,
|
|
"rewards/weighted_rejected": -0.00039333768654614687,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.010468463752944255,
|
|
"grad_norm": 12.950738906860352,
|
|
"learning_rate": 9.895833333333332e-08,
|
|
"logits/chosen": -0.2977302670478821,
|
|
"logits/rejected": -0.3139175474643707,
|
|
"logps/chosen": -294.37579345703125,
|
|
"logps/rejected": -272.7203063964844,
|
|
"logps/weighted_chosen": -2.30804443359375,
|
|
"logps/weighted_rejected": -2.247570753097534,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.296875,
|
|
"rewards/chosen": -0.05908203125,
|
|
"rewards/margins": -0.01513671875,
|
|
"rewards/rejected": -0.0439453125,
|
|
"rewards/weighted_accuracies": 0.34687501192092896,
|
|
"rewards/weighted_chosen": 7.171630568336695e-05,
|
|
"rewards/weighted_margins": 3.662109520519152e-05,
|
|
"rewards/weighted_rejected": 3.509521411615424e-05,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.015702695629416383,
|
|
"grad_norm": 65.14871215820312,
|
|
"learning_rate": 1.5104166666666664e-07,
|
|
"logits/chosen": -0.2917991578578949,
|
|
"logits/rejected": -0.3373458981513977,
|
|
"logps/chosen": -297.9375,
|
|
"logps/rejected": -268.0062561035156,
|
|
"logps/weighted_chosen": -2.3156495094299316,
|
|
"logps/weighted_rejected": -2.529247999191284,
|
|
"loss": 0.693,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": 0.02177734300494194,
|
|
"rewards/margins": -0.06005859375,
|
|
"rewards/rejected": 0.08183594048023224,
|
|
"rewards/weighted_accuracies": 0.34062498807907104,
|
|
"rewards/weighted_chosen": 0.0012060165172442794,
|
|
"rewards/weighted_margins": -0.0021120072342455387,
|
|
"rewards/weighted_rejected": 0.0033180236350744963,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.02093692750588851,
|
|
"grad_norm": 30.699859619140625,
|
|
"learning_rate": 2.03125e-07,
|
|
"logits/chosen": -0.301095575094223,
|
|
"logits/rejected": -0.3433547914028168,
|
|
"logps/chosen": -278.7124938964844,
|
|
"logps/rejected": -253.84530639648438,
|
|
"logps/weighted_chosen": -2.3508667945861816,
|
|
"logps/weighted_rejected": -2.55126953125,
|
|
"loss": 0.6909,
|
|
"rewards/accuracies": 0.2718749940395355,
|
|
"rewards/chosen": -0.014892578125,
|
|
"rewards/margins": -0.105224609375,
|
|
"rewards/rejected": 0.09033203125,
|
|
"rewards/weighted_accuracies": 0.359375,
|
|
"rewards/weighted_chosen": 0.0031726837623864412,
|
|
"rewards/weighted_margins": 0.0010789871448650956,
|
|
"rewards/weighted_rejected": 0.002093696501106024,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.02617115938236064,
|
|
"grad_norm": 15.275498390197754,
|
|
"learning_rate": 2.552083333333333e-07,
|
|
"logits/chosen": -0.28184205293655396,
|
|
"logits/rejected": -0.32042425870895386,
|
|
"logps/chosen": -280.19140625,
|
|
"logps/rejected": -267.4398498535156,
|
|
"logps/weighted_chosen": -2.281604051589966,
|
|
"logps/weighted_rejected": -2.278125047683716,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": 0.009082031436264515,
|
|
"rewards/margins": -0.11552734673023224,
|
|
"rewards/rejected": 0.12460937350988388,
|
|
"rewards/weighted_accuracies": 0.375,
|
|
"rewards/weighted_chosen": 0.0036834715865552425,
|
|
"rewards/weighted_margins": 0.0022117614280432463,
|
|
"rewards/weighted_rejected": 0.0014717101585119963,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.031405391258832765,
|
|
"grad_norm": 42.36075973510742,
|
|
"learning_rate": 3.0729166666666665e-07,
|
|
"logits/chosen": -0.3147949278354645,
|
|
"logits/rejected": -0.3081672787666321,
|
|
"logps/chosen": -277.5492248535156,
|
|
"logps/rejected": -261.63201904296875,
|
|
"logps/weighted_chosen": -2.6925292015075684,
|
|
"logps/weighted_rejected": -2.5440917015075684,
|
|
"loss": 0.6888,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.15625,
|
|
"rewards/margins": 0.06132812425494194,
|
|
"rewards/rejected": 0.09492187201976776,
|
|
"rewards/weighted_accuracies": 0.43437498807907104,
|
|
"rewards/weighted_chosen": 0.0014663696056231856,
|
|
"rewards/weighted_margins": 0.0070289610885083675,
|
|
"rewards/weighted_rejected": -0.005612182430922985,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.036639623135304895,
|
|
"grad_norm": 70.38953399658203,
|
|
"learning_rate": 3.59375e-07,
|
|
"logits/chosen": -0.31798094511032104,
|
|
"logits/rejected": -0.32500457763671875,
|
|
"logps/chosen": -289.6695251464844,
|
|
"logps/rejected": -244.7351531982422,
|
|
"logps/weighted_chosen": -2.19110107421875,
|
|
"logps/weighted_rejected": -2.427661180496216,
|
|
"loss": 0.6869,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.36865234375,
|
|
"rewards/margins": 0.34619140625,
|
|
"rewards/rejected": 0.0224609375,
|
|
"rewards/weighted_accuracies": 0.5375000238418579,
|
|
"rewards/weighted_chosen": 0.02108154259622097,
|
|
"rewards/weighted_margins": 0.011628913693130016,
|
|
"rewards/weighted_rejected": 0.009454727172851562,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.04187385501177702,
|
|
"grad_norm": 54.81600570678711,
|
|
"learning_rate": 4.114583333333333e-07,
|
|
"logits/chosen": -0.2848251461982727,
|
|
"logits/rejected": -0.3061889708042145,
|
|
"logps/chosen": -289.08203125,
|
|
"logps/rejected": -269.95782470703125,
|
|
"logps/weighted_chosen": -2.3393311500549316,
|
|
"logps/weighted_rejected": -2.5093994140625,
|
|
"loss": 0.6827,
|
|
"rewards/accuracies": 0.44999998807907104,
|
|
"rewards/chosen": 0.679980456829071,
|
|
"rewards/margins": 0.3672851622104645,
|
|
"rewards/rejected": 0.31269532442092896,
|
|
"rewards/weighted_accuracies": 0.5062500238418579,
|
|
"rewards/weighted_chosen": 0.048927437514066696,
|
|
"rewards/weighted_margins": 0.025808846578001976,
|
|
"rewards/weighted_rejected": 0.02311401441693306,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.04710808688824915,
|
|
"grad_norm": 30.603567123413086,
|
|
"learning_rate": 4.6354166666666664e-07,
|
|
"logits/chosen": -0.312631219625473,
|
|
"logits/rejected": -0.30586013197898865,
|
|
"logps/chosen": -279.92578125,
|
|
"logps/rejected": -257.5234375,
|
|
"logps/weighted_chosen": -2.5583739280700684,
|
|
"logps/weighted_rejected": -2.6037840843200684,
|
|
"loss": 0.6752,
|
|
"rewards/accuracies": 0.49687498807907104,
|
|
"rewards/chosen": 0.757617175579071,
|
|
"rewards/margins": 0.5347656011581421,
|
|
"rewards/rejected": 0.22285155951976776,
|
|
"rewards/weighted_accuracies": 0.546875,
|
|
"rewards/weighted_chosen": 0.08651771396398544,
|
|
"rewards/weighted_margins": 0.05735473707318306,
|
|
"rewards/weighted_rejected": 0.02917785570025444,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.05234231876472128,
|
|
"grad_norm": 99.52488708496094,
|
|
"learning_rate": 5.156249999999999e-07,
|
|
"logits/chosen": -0.2793869078159332,
|
|
"logits/rejected": -0.3509994447231293,
|
|
"logps/chosen": -290.19061279296875,
|
|
"logps/rejected": -285.94842529296875,
|
|
"logps/weighted_chosen": -2.013537645339966,
|
|
"logps/weighted_rejected": -2.5231690406799316,
|
|
"loss": 0.6984,
|
|
"rewards/accuracies": 0.512499988079071,
|
|
"rewards/chosen": 0.72607421875,
|
|
"rewards/margins": 0.80712890625,
|
|
"rewards/rejected": -0.0810546875,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": 0.07289886474609375,
|
|
"rewards/weighted_margins": 0.019171524792909622,
|
|
"rewards/weighted_rejected": 0.05372200161218643,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.05757655064119341,
|
|
"grad_norm": 55.06239318847656,
|
|
"learning_rate": 5.677083333333333e-07,
|
|
"logits/chosen": -0.3277145326137543,
|
|
"logits/rejected": -0.3189544677734375,
|
|
"logps/chosen": -296.17889404296875,
|
|
"logps/rejected": -261.51953125,
|
|
"logps/weighted_chosen": -2.647631883621216,
|
|
"logps/weighted_rejected": -2.6458740234375,
|
|
"loss": 0.6844,
|
|
"rewards/accuracies": 0.518750011920929,
|
|
"rewards/chosen": 0.1376953125,
|
|
"rewards/margins": 0.789746105670929,
|
|
"rewards/rejected": -0.6522461175918579,
|
|
"rewards/weighted_accuracies": 0.565625011920929,
|
|
"rewards/weighted_chosen": 0.05432138592004776,
|
|
"rewards/weighted_margins": 0.03696594387292862,
|
|
"rewards/weighted_rejected": 0.01735992357134819,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.06281078251766553,
|
|
"grad_norm": 42.45036315917969,
|
|
"learning_rate": 6.197916666666666e-07,
|
|
"logits/chosen": -0.33232802152633667,
|
|
"logits/rejected": -0.35865744948387146,
|
|
"logps/chosen": -294.6078186035156,
|
|
"logps/rejected": -255.01327514648438,
|
|
"logps/weighted_chosen": -1.970422387123108,
|
|
"logps/weighted_rejected": -2.1345458030700684,
|
|
"loss": 0.6837,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": 0.0013671874767169356,
|
|
"rewards/margins": 1.2595703601837158,
|
|
"rewards/rejected": -1.258203148841858,
|
|
"rewards/weighted_accuracies": 0.59375,
|
|
"rewards/weighted_chosen": 0.048947714269161224,
|
|
"rewards/weighted_margins": 0.03674011304974556,
|
|
"rewards/weighted_rejected": 0.012198830023407936,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.06804501439413765,
|
|
"grad_norm": 28.637327194213867,
|
|
"learning_rate": 6.718749999999999e-07,
|
|
"logits/chosen": -0.2933105528354645,
|
|
"logits/rejected": -0.29020920395851135,
|
|
"logps/chosen": -305.86602783203125,
|
|
"logps/rejected": -279.09844970703125,
|
|
"logps/weighted_chosen": -1.9946777820587158,
|
|
"logps/weighted_rejected": -2.2933349609375,
|
|
"loss": 0.6754,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.285400390625,
|
|
"rewards/margins": 1.7852051258087158,
|
|
"rewards/rejected": -1.4998047351837158,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": 0.06223602220416069,
|
|
"rewards/weighted_margins": 0.05499267578125,
|
|
"rewards/weighted_rejected": 0.0072036744095385075,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.07327924627060979,
|
|
"grad_norm": 20.866477966308594,
|
|
"learning_rate": 7.239583333333333e-07,
|
|
"logits/chosen": -0.33005982637405396,
|
|
"logits/rejected": -0.34638214111328125,
|
|
"logps/chosen": -300.7281188964844,
|
|
"logps/rejected": -276.9156188964844,
|
|
"logps/weighted_chosen": -2.294506788253784,
|
|
"logps/weighted_rejected": -2.561084032058716,
|
|
"loss": 0.6606,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -1.998437523841858,
|
|
"rewards/margins": 2.3173828125,
|
|
"rewards/rejected": -4.315820217132568,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": 0.04616241529583931,
|
|
"rewards/weighted_margins": 0.08864898979663849,
|
|
"rewards/weighted_rejected": -0.04253540188074112,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.07851347814708191,
|
|
"grad_norm": 43.497623443603516,
|
|
"learning_rate": 7.760416666666666e-07,
|
|
"logits/chosen": -0.2890525758266449,
|
|
"logits/rejected": -0.30130767822265625,
|
|
"logps/chosen": -287.3109436035156,
|
|
"logps/rejected": -256.72967529296875,
|
|
"logps/weighted_chosen": -2.883862257003784,
|
|
"logps/weighted_rejected": -2.796630859375,
|
|
"loss": 0.664,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -3.108203172683716,
|
|
"rewards/margins": 2.7275390625,
|
|
"rewards/rejected": -5.835741996765137,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": 0.04738159105181694,
|
|
"rewards/weighted_margins": 0.09773864597082138,
|
|
"rewards/weighted_rejected": -0.05034027248620987,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.08374771002355404,
|
|
"grad_norm": 12.989988327026367,
|
|
"learning_rate": 8.28125e-07,
|
|
"logits/chosen": -0.3334007263183594,
|
|
"logits/rejected": -0.3831237852573395,
|
|
"logps/chosen": -308.2320251464844,
|
|
"logps/rejected": -281.5335998535156,
|
|
"logps/weighted_chosen": -2.378857374191284,
|
|
"logps/weighted_rejected": -2.3824095726013184,
|
|
"loss": 0.6464,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -4.347460746765137,
|
|
"rewards/margins": 3.98388671875,
|
|
"rewards/rejected": -8.331347465515137,
|
|
"rewards/weighted_accuracies": 0.65625,
|
|
"rewards/weighted_chosen": 0.04017028957605362,
|
|
"rewards/weighted_margins": 0.15716858208179474,
|
|
"rewards/weighted_rejected": -0.11710510402917862,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.08898194190002617,
|
|
"grad_norm": 691.15625,
|
|
"learning_rate": 8.802083333333333e-07,
|
|
"logits/chosen": -0.37783128023147583,
|
|
"logits/rejected": -0.37789613008499146,
|
|
"logps/chosen": -310.76953125,
|
|
"logps/rejected": -270.8374938964844,
|
|
"logps/weighted_chosen": -2.6180419921875,
|
|
"logps/weighted_rejected": -2.8051514625549316,
|
|
"loss": 0.683,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -7.780859470367432,
|
|
"rewards/margins": 4.9013671875,
|
|
"rewards/rejected": -12.683398246765137,
|
|
"rewards/weighted_accuracies": 0.643750011920929,
|
|
"rewards/weighted_chosen": 0.06108245998620987,
|
|
"rewards/weighted_margins": 0.17230224609375,
|
|
"rewards/weighted_rejected": -0.11125946044921875,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.0942161737764983,
|
|
"grad_norm": 65.56312561035156,
|
|
"learning_rate": 9.322916666666666e-07,
|
|
"logits/chosen": -0.3433074951171875,
|
|
"logits/rejected": -0.363290399312973,
|
|
"logps/chosen": -285.8812561035156,
|
|
"logps/rejected": -274.94842529296875,
|
|
"logps/weighted_chosen": -2.50494384765625,
|
|
"logps/weighted_rejected": -2.9246826171875,
|
|
"loss": 0.6821,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -12.179101943969727,
|
|
"rewards/margins": 6.432421684265137,
|
|
"rewards/rejected": -18.608592987060547,
|
|
"rewards/weighted_accuracies": 0.59375,
|
|
"rewards/weighted_chosen": -0.0216827392578125,
|
|
"rewards/weighted_margins": 0.19813232123851776,
|
|
"rewards/weighted_rejected": -0.2197723388671875,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.09945040565297043,
|
|
"grad_norm": 26.828750610351562,
|
|
"learning_rate": 9.84375e-07,
|
|
"logits/chosen": -0.36560744047164917,
|
|
"logits/rejected": -0.40380173921585083,
|
|
"logps/chosen": -321.97967529296875,
|
|
"logps/rejected": -287.5570373535156,
|
|
"logps/weighted_chosen": -2.3037109375,
|
|
"logps/weighted_rejected": -2.64697265625,
|
|
"loss": 0.6489,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -16.246679306030273,
|
|
"rewards/margins": 7.440039157867432,
|
|
"rewards/rejected": -23.685937881469727,
|
|
"rewards/weighted_accuracies": 0.640625,
|
|
"rewards/weighted_chosen": -0.11536254733800888,
|
|
"rewards/weighted_margins": 0.19368895888328552,
|
|
"rewards/weighted_rejected": -0.309121698141098,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.10468463752944256,
|
|
"grad_norm": 117.84883880615234,
|
|
"learning_rate": 9.99959085414323e-07,
|
|
"logits/chosen": -0.4036270081996918,
|
|
"logits/rejected": -0.43666380643844604,
|
|
"logps/chosen": -327.8265686035156,
|
|
"logps/rejected": -284.4234313964844,
|
|
"logps/weighted_chosen": -2.6368165016174316,
|
|
"logps/weighted_rejected": -3.0156006813049316,
|
|
"loss": 0.6383,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -18.538379669189453,
|
|
"rewards/margins": 8.212695121765137,
|
|
"rewards/rejected": -26.748046875,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.12391586601734161,
|
|
"rewards/weighted_margins": 0.28039854764938354,
|
|
"rewards/weighted_rejected": -0.404205322265625,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.10991886940591468,
|
|
"grad_norm": 15.525572776794434,
|
|
"learning_rate": 9.997587035630105e-07,
|
|
"logits/chosen": -0.41632765531539917,
|
|
"logits/rejected": -0.4597106873989105,
|
|
"logps/chosen": -308.75469970703125,
|
|
"logps/rejected": -316.69842529296875,
|
|
"logps/weighted_chosen": -2.439282178878784,
|
|
"logps/weighted_rejected": -3.152050733566284,
|
|
"loss": 0.6437,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -21.632617950439453,
|
|
"rewards/margins": 10.560155868530273,
|
|
"rewards/rejected": -32.18867111206055,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.2110244780778885,
|
|
"rewards/weighted_margins": 0.23976440727710724,
|
|
"rewards/weighted_rejected": -0.4508819580078125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.11515310128238682,
|
|
"grad_norm": 31.583986282348633,
|
|
"learning_rate": 9.99391406364405e-07,
|
|
"logits/chosen": -0.37994080781936646,
|
|
"logits/rejected": -0.384744256734848,
|
|
"logps/chosen": -318.63592529296875,
|
|
"logps/rejected": -305.0,
|
|
"logps/weighted_chosen": -2.9573731422424316,
|
|
"logps/weighted_rejected": -3.207348585128784,
|
|
"loss": 0.7259,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -26.522266387939453,
|
|
"rewards/margins": 9.657129287719727,
|
|
"rewards/rejected": -36.18437576293945,
|
|
"rewards/weighted_accuracies": 0.625,
|
|
"rewards/weighted_chosen": -0.3494918942451477,
|
|
"rewards/weighted_margins": 0.17980042099952698,
|
|
"rewards/weighted_rejected": -0.5295776128768921,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.12038733315885894,
|
|
"grad_norm": 611.2006225585938,
|
|
"learning_rate": 9.988573164927884e-07,
|
|
"logits/chosen": -0.3254447877407074,
|
|
"logits/rejected": -0.36442261934280396,
|
|
"logps/chosen": -297.16717529296875,
|
|
"logps/rejected": -296.48907470703125,
|
|
"logps/weighted_chosen": -2.35888671875,
|
|
"logps/weighted_rejected": -2.804126024246216,
|
|
"loss": 0.6637,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -28.293750762939453,
|
|
"rewards/margins": 16.522851943969727,
|
|
"rewards/rejected": -44.817970275878906,
|
|
"rewards/weighted_accuracies": 0.6812499761581421,
|
|
"rewards/weighted_chosen": -0.1478603333234787,
|
|
"rewards/weighted_margins": 0.3882904052734375,
|
|
"rewards/weighted_rejected": -0.5360336303710938,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.12562156503533106,
|
|
"grad_norm": 34.738059997558594,
|
|
"learning_rate": 9.98156612329838e-07,
|
|
"logits/chosen": -0.3876676559448242,
|
|
"logits/rejected": -0.4364303648471832,
|
|
"logps/chosen": -312.5921936035156,
|
|
"logps/rejected": -348.60626220703125,
|
|
"logps/weighted_chosen": -2.594775438308716,
|
|
"logps/weighted_rejected": -3.1068115234375,
|
|
"loss": 0.7482,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -47.75507736206055,
|
|
"rewards/margins": 18.762109756469727,
|
|
"rewards/rejected": -66.5289077758789,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.319406121969223,
|
|
"rewards/weighted_margins": 0.204498291015625,
|
|
"rewards/weighted_rejected": -0.5241454839706421,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.13085579691180318,
|
|
"grad_norm": 141.2784423828125,
|
|
"learning_rate": 9.97289527905053e-07,
|
|
"logits/chosen": -0.4159797728061676,
|
|
"logits/rejected": -0.4382568299770355,
|
|
"logps/chosen": -308.44842529296875,
|
|
"logps/rejected": -313.49530029296875,
|
|
"logps/weighted_chosen": -2.802807569503784,
|
|
"logps/weighted_rejected": -2.9874024391174316,
|
|
"loss": 0.7583,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -44.16621017456055,
|
|
"rewards/margins": 13.382226943969727,
|
|
"rewards/rejected": -57.54296875,
|
|
"rewards/weighted_accuracies": 0.6187499761581421,
|
|
"rewards/weighted_chosen": -0.20551452040672302,
|
|
"rewards/weighted_margins": 0.22150878608226776,
|
|
"rewards/weighted_rejected": -0.42730408906936646,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.1360900287882753,
|
|
"grad_norm": 75.63247680664062,
|
|
"learning_rate": 9.962563528175875e-07,
|
|
"logits/chosen": -0.4156021177768707,
|
|
"logits/rejected": -0.4493362307548523,
|
|
"logps/chosen": -342.1312561035156,
|
|
"logps/rejected": -317.84063720703125,
|
|
"logps/weighted_chosen": -2.56585693359375,
|
|
"logps/weighted_rejected": -3.4276366233825684,
|
|
"loss": 0.6673,
|
|
"rewards/accuracies": 0.5625,
|
|
"rewards/chosen": -43.44648361206055,
|
|
"rewards/margins": 13.459375381469727,
|
|
"rewards/rejected": -56.90898513793945,
|
|
"rewards/weighted_accuracies": 0.6156250238418579,
|
|
"rewards/weighted_chosen": -0.310464471578598,
|
|
"rewards/weighted_margins": 0.32973939180374146,
|
|
"rewards/weighted_rejected": -0.6404876708984375,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.14132426066474746,
|
|
"grad_norm": 32.991607666015625,
|
|
"learning_rate": 9.950574321395277e-07,
|
|
"logits/chosen": -0.507153332233429,
|
|
"logits/rejected": -0.5264068841934204,
|
|
"logps/chosen": -317.0335998535156,
|
|
"logps/rejected": -300.5718688964844,
|
|
"logps/weighted_chosen": -2.669018507003784,
|
|
"logps/weighted_rejected": -2.8846678733825684,
|
|
"loss": 0.6774,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -27.629491806030273,
|
|
"rewards/margins": 9.762109756469727,
|
|
"rewards/rejected": -37.39765548706055,
|
|
"rewards/weighted_accuracies": 0.6000000238418579,
|
|
"rewards/weighted_chosen": -0.23316803574562073,
|
|
"rewards/weighted_margins": 0.2321929931640625,
|
|
"rewards/weighted_rejected": -0.46527099609375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.14655849254121958,
|
|
"grad_norm": 41.87471389770508,
|
|
"learning_rate": 9.936931663006413e-07,
|
|
"logits/chosen": -0.5015045404434204,
|
|
"logits/rejected": -0.49525564908981323,
|
|
"logps/chosen": -319.9296875,
|
|
"logps/rejected": -312.28826904296875,
|
|
"logps/weighted_chosen": -2.667797803878784,
|
|
"logps/weighted_rejected": -3.121386766433716,
|
|
"loss": 0.6458,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -17.077342987060547,
|
|
"rewards/margins": 15.751367568969727,
|
|
"rewards/rejected": -32.82636642456055,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -0.10239715874195099,
|
|
"rewards/weighted_margins": 0.3067169189453125,
|
|
"rewards/weighted_rejected": -0.409027099609375,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.1517927244176917,
|
|
"grad_norm": 22.655261993408203,
|
|
"learning_rate": 9.921640109546357e-07,
|
|
"logits/chosen": -0.4936012327671051,
|
|
"logits/rejected": -0.5640270113945007,
|
|
"logps/chosen": -286.4117126464844,
|
|
"logps/rejected": -286.15313720703125,
|
|
"logps/weighted_chosen": -2.640380859375,
|
|
"logps/weighted_rejected": -3.774829149246216,
|
|
"loss": 0.6369,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -19.629491806030273,
|
|
"rewards/margins": 14.98291015625,
|
|
"rewards/rejected": -34.61503982543945,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.21005859971046448,
|
|
"rewards/weighted_margins": 0.3854110836982727,
|
|
"rewards/weighted_rejected": -0.5955780148506165,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.15702695629416383,
|
|
"grad_norm": 33.83458709716797,
|
|
"learning_rate": 9.90470476826975e-07,
|
|
"logits/chosen": -0.5687958002090454,
|
|
"logits/rejected": -0.5700439214706421,
|
|
"logps/chosen": -298.06561279296875,
|
|
"logps/rejected": -311.8359375,
|
|
"logps/weighted_chosen": -2.555248975753784,
|
|
"logps/weighted_rejected": -3.0333251953125,
|
|
"loss": 0.6326,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -28.754297256469727,
|
|
"rewards/margins": 15.813085556030273,
|
|
"rewards/rejected": -44.567771911621094,
|
|
"rewards/weighted_accuracies": 0.6468750238418579,
|
|
"rewards/weighted_chosen": -0.25576478242874146,
|
|
"rewards/weighted_margins": 0.3327178955078125,
|
|
"rewards/weighted_rejected": -0.5886474847793579,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.16226118817063595,
|
|
"grad_norm": 29.578954696655273,
|
|
"learning_rate": 9.886131295443002e-07,
|
|
"logits/chosen": -0.6218292117118835,
|
|
"logits/rejected": -0.6808746457099915,
|
|
"logps/chosen": -313.44061279296875,
|
|
"logps/rejected": -297.9156188964844,
|
|
"logps/weighted_chosen": -2.719403028488159,
|
|
"logps/weighted_rejected": -3.1142334938049316,
|
|
"loss": 0.6221,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": -32.1875,
|
|
"rewards/margins": 15.571874618530273,
|
|
"rewards/rejected": -47.7666015625,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.19769592583179474,
|
|
"rewards/weighted_margins": 0.521374523639679,
|
|
"rewards/weighted_rejected": -0.7188171148300171,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.16749542004710807,
|
|
"grad_norm": 15.948070526123047,
|
|
"learning_rate": 9.865925894455166e-07,
|
|
"logits/chosen": -0.6636382937431335,
|
|
"logits/rejected": -0.6817939877510071,
|
|
"logps/chosen": -336.1507873535156,
|
|
"logps/rejected": -306.62890625,
|
|
"logps/weighted_chosen": -2.83941650390625,
|
|
"logps/weighted_rejected": -3.273486375808716,
|
|
"loss": 0.7046,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -36.21074295043945,
|
|
"rewards/margins": 12.9765625,
|
|
"rewards/rejected": -49.17656326293945,
|
|
"rewards/weighted_accuracies": 0.6343749761581421,
|
|
"rewards/weighted_chosen": -0.3679946959018707,
|
|
"rewards/weighted_margins": 0.278738409280777,
|
|
"rewards/weighted_rejected": -0.6464904546737671,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.17272965192358022,
|
|
"grad_norm": 53.10931396484375,
|
|
"learning_rate": 9.84409531374603e-07,
|
|
"logits/chosen": -0.6222923398017883,
|
|
"logits/rejected": -0.607867419719696,
|
|
"logps/chosen": -344.19061279296875,
|
|
"logps/rejected": -315.84686279296875,
|
|
"logps/weighted_chosen": -2.915234327316284,
|
|
"logps/weighted_rejected": -3.346630811691284,
|
|
"loss": 0.6334,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -39.39531326293945,
|
|
"rewards/margins": 13.606640815734863,
|
|
"rewards/rejected": -53.00273513793945,
|
|
"rewards/weighted_accuracies": 0.65625,
|
|
"rewards/weighted_chosen": -0.33662718534469604,
|
|
"rewards/weighted_margins": 0.3510375916957855,
|
|
"rewards/weighted_rejected": -0.6879855990409851,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.17796388380005235,
|
|
"grad_norm": 13.994344711303711,
|
|
"learning_rate": 9.820646844552219e-07,
|
|
"logits/chosen": -0.6615959405899048,
|
|
"logits/rejected": -0.714038074016571,
|
|
"logps/chosen": -312.7328186035156,
|
|
"logps/rejected": -323.3359375,
|
|
"logps/weighted_chosen": -2.910736083984375,
|
|
"logps/weighted_rejected": -3.2298340797424316,
|
|
"loss": 0.6376,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -36.327537536621094,
|
|
"rewards/margins": 21.465625762939453,
|
|
"rewards/rejected": -57.79179763793945,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.34371185302734375,
|
|
"rewards/weighted_margins": 0.390716552734375,
|
|
"rewards/weighted_rejected": -0.734301745891571,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.18319811567652447,
|
|
"grad_norm": 13.682384490966797,
|
|
"learning_rate": 9.795588318471964e-07,
|
|
"logits/chosen": -0.774444580078125,
|
|
"logits/rejected": -0.7887184023857117,
|
|
"logps/chosen": -302.76251220703125,
|
|
"logps/rejected": -336.90313720703125,
|
|
"logps/weighted_chosen": -2.718432664871216,
|
|
"logps/weighted_rejected": -3.0622315406799316,
|
|
"loss": 0.6451,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -43.29413986206055,
|
|
"rewards/margins": 17.229297637939453,
|
|
"rewards/rejected": -60.52812576293945,
|
|
"rewards/weighted_accuracies": 0.637499988079071,
|
|
"rewards/weighted_chosen": -0.33645325899124146,
|
|
"rewards/weighted_margins": 0.33460694551467896,
|
|
"rewards/weighted_rejected": -0.6709426641464233,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.1884323475529966,
|
|
"grad_norm": 19.35496711730957,
|
|
"learning_rate": 9.768928104849415e-07,
|
|
"logits/chosen": -0.815447986125946,
|
|
"logits/rejected": -0.8216186761856079,
|
|
"logps/chosen": -330.66015625,
|
|
"logps/rejected": -313.8687438964844,
|
|
"logps/weighted_chosen": -2.970752000808716,
|
|
"logps/weighted_rejected": -3.1889405250549316,
|
|
"loss": 0.6763,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -47.90937423706055,
|
|
"rewards/margins": 17.064844131469727,
|
|
"rewards/rejected": -64.98515319824219,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.3897460997104645,
|
|
"rewards/weighted_margins": 0.384918212890625,
|
|
"rewards/weighted_rejected": -0.7745300531387329,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.19366657942946872,
|
|
"grad_norm": 21.70383071899414,
|
|
"learning_rate": 9.740675107979355e-07,
|
|
"logits/chosen": -0.7939224243164062,
|
|
"logits/rejected": -0.827624499797821,
|
|
"logps/chosen": -373.0406188964844,
|
|
"logps/rejected": -349.3343811035156,
|
|
"logps/weighted_chosen": -2.506054639816284,
|
|
"logps/weighted_rejected": -3.43896484375,
|
|
"loss": 0.6215,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -58.082420349121094,
|
|
"rewards/margins": 16.594335556030273,
|
|
"rewards/rejected": -74.671875,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.48696595430374146,
|
|
"rewards/weighted_margins": 0.412750244140625,
|
|
"rewards/weighted_rejected": -0.90020751953125,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.19890081130594087,
|
|
"grad_norm": 21.682790756225586,
|
|
"learning_rate": 9.71083876413323e-07,
|
|
"logits/chosen": -0.77825927734375,
|
|
"logits/rejected": -0.7921401858329773,
|
|
"logps/chosen": -367.0062561035156,
|
|
"logps/rejected": -355.78594970703125,
|
|
"logps/weighted_chosen": -2.644775390625,
|
|
"logps/weighted_rejected": -3.5374755859375,
|
|
"loss": 0.6609,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -64.59492492675781,
|
|
"rewards/margins": 21.614843368530273,
|
|
"rewards/rejected": -86.1988296508789,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.5685027837753296,
|
|
"rewards/weighted_margins": 0.3586792051792145,
|
|
"rewards/weighted_rejected": -0.9269439578056335,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.204135043182413,
|
|
"grad_norm": 30.26679229736328,
|
|
"learning_rate": 9.67942903840751e-07,
|
|
"logits/chosen": -0.8401764035224915,
|
|
"logits/rejected": -0.898754894733429,
|
|
"logps/chosen": -362.0328063964844,
|
|
"logps/rejected": -362.3687438964844,
|
|
"logps/weighted_chosen": -2.8179688453674316,
|
|
"logps/weighted_rejected": -3.4515624046325684,
|
|
"loss": 0.5992,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -57.87226486206055,
|
|
"rewards/margins": 30.196680068969727,
|
|
"rewards/rejected": -88.0804672241211,
|
|
"rewards/weighted_accuracies": 0.715624988079071,
|
|
"rewards/weighted_chosen": -0.456369012594223,
|
|
"rewards/weighted_margins": 0.495849609375,
|
|
"rewards/weighted_rejected": -0.951983630657196,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.2093692750588851,
|
|
"grad_norm": 37.81236267089844,
|
|
"learning_rate": 9.646456421395447e-07,
|
|
"logits/chosen": -0.8902359008789062,
|
|
"logits/rejected": -0.9107666015625,
|
|
"logps/chosen": -380.6343688964844,
|
|
"logps/rejected": -399.7281188964844,
|
|
"logps/weighted_chosen": -2.709228515625,
|
|
"logps/weighted_rejected": -3.455004930496216,
|
|
"loss": 0.6557,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -62.43437576293945,
|
|
"rewards/margins": 33.634765625,
|
|
"rewards/rejected": -96.0941390991211,
|
|
"rewards/weighted_accuracies": 0.653124988079071,
|
|
"rewards/weighted_chosen": -0.4688262939453125,
|
|
"rewards/weighted_margins": 0.3546081483364105,
|
|
"rewards/weighted_rejected": -0.8232513666152954,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.21460350693535724,
|
|
"grad_norm": 21.761716842651367,
|
|
"learning_rate": 9.611931925683266e-07,
|
|
"logits/chosen": -0.849993109703064,
|
|
"logits/rejected": -0.8886367678642273,
|
|
"logps/chosen": -368.8374938964844,
|
|
"logps/rejected": -353.73126220703125,
|
|
"logps/weighted_chosen": -2.533374071121216,
|
|
"logps/weighted_rejected": -3.233935594558716,
|
|
"loss": 0.591,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -62.575782775878906,
|
|
"rewards/margins": 27.423242568969727,
|
|
"rewards/rejected": -90.0132827758789,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.4288879334926605,
|
|
"rewards/weighted_margins": 0.507708728313446,
|
|
"rewards/weighted_rejected": -0.9364013671875,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.21983773881182936,
|
|
"grad_norm": 28.842378616333008,
|
|
"learning_rate": 9.575867082172085e-07,
|
|
"logits/chosen": -0.836225152015686,
|
|
"logits/rejected": -0.8854034543037415,
|
|
"logps/chosen": -371.90625,
|
|
"logps/rejected": -369.4125061035156,
|
|
"logps/weighted_chosen": -2.9190917015075684,
|
|
"logps/weighted_rejected": -3.118579149246216,
|
|
"loss": 0.6198,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -66.98905944824219,
|
|
"rewards/margins": 32.337890625,
|
|
"rewards/rejected": -99.3402328491211,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.4911743104457855,
|
|
"rewards/weighted_margins": 0.5335143804550171,
|
|
"rewards/weighted_rejected": -1.0242736339569092,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.22507197068830148,
|
|
"grad_norm": 16.805343627929688,
|
|
"learning_rate": 9.538273936226673e-07,
|
|
"logits/chosen": -0.8646026849746704,
|
|
"logits/rejected": -0.898516833782196,
|
|
"logps/chosen": -328.94921875,
|
|
"logps/rejected": -353.66717529296875,
|
|
"logps/weighted_chosen": -3.1983399391174316,
|
|
"logps/weighted_rejected": -3.6509766578674316,
|
|
"loss": 0.6173,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -63.52910232543945,
|
|
"rewards/margins": 25.6201171875,
|
|
"rewards/rejected": -89.1246109008789,
|
|
"rewards/weighted_accuracies": 0.6499999761581421,
|
|
"rewards/weighted_chosen": -0.436279296875,
|
|
"rewards/weighted_margins": 0.438516229391098,
|
|
"rewards/weighted_rejected": -0.8747711181640625,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.23030620256477363,
|
|
"grad_norm": 23.78999137878418,
|
|
"learning_rate": 9.499165043652391e-07,
|
|
"logits/chosen": -0.8759521245956421,
|
|
"logits/rejected": -0.8865814208984375,
|
|
"logps/chosen": -358.69140625,
|
|
"logps/rejected": -358.76251220703125,
|
|
"logps/weighted_chosen": -3.2625732421875,
|
|
"logps/weighted_rejected": -3.359301805496216,
|
|
"loss": 0.6366,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -67.4326171875,
|
|
"rewards/margins": 24.610157012939453,
|
|
"rewards/rejected": -92.03788757324219,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.5896576046943665,
|
|
"rewards/weighted_margins": 0.34770506620407104,
|
|
"rewards/weighted_rejected": -0.9376128911972046,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.23554043444124576,
|
|
"grad_norm": 49.646263122558594,
|
|
"learning_rate": 9.458553466501665e-07,
|
|
"logits/chosen": -0.9158996343612671,
|
|
"logits/rejected": -0.9590820074081421,
|
|
"logps/chosen": -355.3109436035156,
|
|
"logps/rejected": -341.9765625,
|
|
"logps/weighted_chosen": -3.231823682785034,
|
|
"logps/weighted_rejected": -3.4568848609924316,
|
|
"loss": 0.657,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -69.14453125,
|
|
"rewards/margins": 28.884180068969727,
|
|
"rewards/rejected": -98.0492172241211,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.6470184326171875,
|
|
"rewards/weighted_margins": 0.4533935487270355,
|
|
"rewards/weighted_rejected": -1.100683569908142,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.24077466631771788,
|
|
"grad_norm": 38.45325469970703,
|
|
"learning_rate": 9.416452768711366e-07,
|
|
"logits/chosen": -0.9281005859375,
|
|
"logits/rejected": -0.9545837640762329,
|
|
"logps/chosen": -375.6390686035156,
|
|
"logps/rejected": -370.203125,
|
|
"logps/weighted_chosen": -2.9860596656799316,
|
|
"logps/weighted_rejected": -3.635498046875,
|
|
"loss": 0.6457,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -83.3871078491211,
|
|
"rewards/margins": 26.872264862060547,
|
|
"rewards/rejected": -110.26094055175781,
|
|
"rewards/weighted_accuracies": 0.6625000238418579,
|
|
"rewards/weighted_chosen": -0.6367782354354858,
|
|
"rewards/weighted_margins": 0.515301525592804,
|
|
"rewards/weighted_rejected": -1.152197241783142,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.24600889819419,
|
|
"grad_norm": 59.67642593383789,
|
|
"learning_rate": 9.372877011572557e-07,
|
|
"logits/chosen": -0.8797394037246704,
|
|
"logits/rejected": -0.904949963092804,
|
|
"logps/chosen": -398.8531188964844,
|
|
"logps/rejected": -388.22186279296875,
|
|
"logps/weighted_chosen": -3.0103516578674316,
|
|
"logps/weighted_rejected": -3.409374952316284,
|
|
"loss": 0.6181,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": -88.2437515258789,
|
|
"rewards/margins": 25.542577743530273,
|
|
"rewards/rejected": -113.7890625,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.6517273187637329,
|
|
"rewards/weighted_margins": 0.482818603515625,
|
|
"rewards/weighted_rejected": -1.1340011358261108,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.2512431300706621,
|
|
"grad_norm": 27.465656280517578,
|
|
"learning_rate": 9.327840749034141e-07,
|
|
"logits/chosen": -0.910723865032196,
|
|
"logits/rejected": -0.9439147710800171,
|
|
"logps/chosen": -364.4750061035156,
|
|
"logps/rejected": -392.03436279296875,
|
|
"logps/weighted_chosen": -2.8880372047424316,
|
|
"logps/weighted_rejected": -4.069311618804932,
|
|
"loss": 0.6276,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -77.44355773925781,
|
|
"rewards/margins": 38.147850036621094,
|
|
"rewards/rejected": -115.59648132324219,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.5768798589706421,
|
|
"rewards/weighted_margins": 0.541027843952179,
|
|
"rewards/weighted_rejected": -1.1178162097930908,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.2564773619471343,
|
|
"grad_norm": 34.35914993286133,
|
|
"learning_rate": 9.281359022841965e-07,
|
|
"logits/chosen": -0.80584716796875,
|
|
"logits/rejected": -0.8291991949081421,
|
|
"logps/chosen": -349.86798095703125,
|
|
"logps/rejected": -354.75701904296875,
|
|
"logps/weighted_chosen": -2.914257764816284,
|
|
"logps/weighted_rejected": -4.178680419921875,
|
|
"loss": 0.63,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -74.94511413574219,
|
|
"rewards/margins": 34.957618713378906,
|
|
"rewards/rejected": -109.90644836425781,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.564099133014679,
|
|
"rewards/weighted_margins": 0.658795177936554,
|
|
"rewards/weighted_rejected": -1.2229950428009033,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"grad_norm": 41.01088333129883,
|
|
"learning_rate": 9.233447357514989e-07,
|
|
"logits/chosen": -0.775347888469696,
|
|
"logits/rejected": -0.8226684331893921,
|
|
"logps/chosen": -369.1148376464844,
|
|
"logps/rejected": -373.8374938964844,
|
|
"logps/weighted_chosen": -3.288012742996216,
|
|
"logps/weighted_rejected": -3.7696776390075684,
|
|
"loss": 0.6225,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -71.1283187866211,
|
|
"rewards/margins": 32.867576599121094,
|
|
"rewards/rejected": -104.0218734741211,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -0.5540069341659546,
|
|
"rewards/weighted_margins": 0.598828136920929,
|
|
"rewards/weighted_rejected": -1.1535155773162842,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"eval_logits/chosen": -0.8767929673194885,
|
|
"eval_logits/rejected": -0.8968105316162109,
|
|
"eval_logps/chosen": -369.21099853515625,
|
|
"eval_logps/rejected": -379.3609924316406,
|
|
"eval_logps/weighted_chosen": -2.957455635070801,
|
|
"eval_logps/weighted_rejected": -3.6658689975738525,
|
|
"eval_loss": 0.6115986108779907,
|
|
"eval_rewards/accuracies": 0.6294999718666077,
|
|
"eval_rewards/chosen": -80.43468475341797,
|
|
"eval_rewards/margins": 31.297624588012695,
|
|
"eval_rewards/rejected": -111.73987579345703,
|
|
"eval_rewards/weighted_accuracies": 0.6759999990463257,
|
|
"eval_rewards/weighted_chosen": -0.5706117153167725,
|
|
"eval_rewards/weighted_margins": 0.5400464534759521,
|
|
"eval_rewards/weighted_rejected": -1.1106581687927246,
|
|
"eval_runtime": 1361.4222,
|
|
"eval_samples_per_second": 1.469,
|
|
"eval_steps_per_second": 0.367,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.2669458257000785,
|
|
"grad_norm": 37.983856201171875,
|
|
"learning_rate": 9.184121755160232e-07,
|
|
"logits/chosen": -0.8435028195381165,
|
|
"logits/rejected": -0.8772613406181335,
|
|
"logps/chosen": -378.20782470703125,
|
|
"logps/rejected": -406.3187561035156,
|
|
"logps/weighted_chosen": -3.217089891433716,
|
|
"logps/weighted_rejected": -3.7809815406799316,
|
|
"loss": 0.6028,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -77.0746078491211,
|
|
"rewards/margins": 40.68242263793945,
|
|
"rewards/rejected": -117.759765625,
|
|
"rewards/weighted_accuracies": 0.6875,
|
|
"rewards/weighted_chosen": -0.5556915402412415,
|
|
"rewards/weighted_margins": 0.5749145746231079,
|
|
"rewards/weighted_rejected": -1.130804419517517,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.2721800575765506,
|
|
"grad_norm": 46.08055877685547,
|
|
"learning_rate": 9.133398690128193e-07,
|
|
"logits/chosen": -0.894885241985321,
|
|
"logits/rejected": -0.9157882928848267,
|
|
"logps/chosen": -405.5679626464844,
|
|
"logps/rejected": -415.10076904296875,
|
|
"logps/weighted_chosen": -3.0770506858825684,
|
|
"logps/weighted_rejected": -3.949780225753784,
|
|
"loss": 0.5762,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -84.00956726074219,
|
|
"rewards/margins": 50.470703125,
|
|
"rewards/rejected": -134.46737670898438,
|
|
"rewards/weighted_accuracies": 0.7250000238418579,
|
|
"rewards/weighted_chosen": -0.49931639432907104,
|
|
"rewards/weighted_margins": 0.658154308795929,
|
|
"rewards/weighted_rejected": -1.1574280261993408,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.27741428945302277,
|
|
"grad_norm": 40.89746856689453,
|
|
"learning_rate": 9.081295103510554e-07,
|
|
"logits/chosen": -0.8876678347587585,
|
|
"logits/rejected": -0.9430938959121704,
|
|
"logps/chosen": -383.52734375,
|
|
"logps/rejected": -413.60626220703125,
|
|
"logps/weighted_chosen": -2.752880811691284,
|
|
"logps/weighted_rejected": -4.01953125,
|
|
"loss": 0.5471,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -99.4878921508789,
|
|
"rewards/margins": 44.412109375,
|
|
"rewards/rejected": -143.859375,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.6086364984512329,
|
|
"rewards/weighted_margins": 0.8298507928848267,
|
|
"rewards/weighted_rejected": -1.438207983970642,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.2826485213294949,
|
|
"grad_norm": 27.849748611450195,
|
|
"learning_rate": 9.027828397481989e-07,
|
|
"logits/chosen": -0.8531036376953125,
|
|
"logits/rejected": -0.885467529296875,
|
|
"logps/chosen": -354.4359436035156,
|
|
"logps/rejected": -387.6328125,
|
|
"logps/weighted_chosen": -3.355029344558716,
|
|
"logps/weighted_rejected": -4.168432712554932,
|
|
"loss": 0.5956,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -93.89375305175781,
|
|
"rewards/margins": 38.216407775878906,
|
|
"rewards/rejected": -132.11776733398438,
|
|
"rewards/weighted_accuracies": 0.675000011920929,
|
|
"rewards/weighted_chosen": -0.6640685796737671,
|
|
"rewards/weighted_margins": 0.6357482671737671,
|
|
"rewards/weighted_rejected": -1.2999756336212158,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.287882753205967,
|
|
"grad_norm": 22.531660079956055,
|
|
"learning_rate": 8.973016429487988e-07,
|
|
"logits/chosen": -0.8864593505859375,
|
|
"logits/rejected": -0.9035919308662415,
|
|
"logps/chosen": -375.8890686035156,
|
|
"logps/rejected": -396.171875,
|
|
"logps/weighted_chosen": -3.2917237281799316,
|
|
"logps/weighted_rejected": -3.580859422683716,
|
|
"loss": 0.6356,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -94.04902648925781,
|
|
"rewards/margins": 47.923240661621094,
|
|
"rewards/rejected": -141.9646453857422,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.7129974365234375,
|
|
"rewards/weighted_margins": 0.53741455078125,
|
|
"rewards/weighted_rejected": -1.2503478527069092,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.29311698508243916,
|
|
"grad_norm": 36.68463897705078,
|
|
"learning_rate": 8.916877506280601e-07,
|
|
"logits/chosen": -0.9174438714981079,
|
|
"logits/rejected": -0.9228302240371704,
|
|
"logps/chosen": -387.2203063964844,
|
|
"logps/rejected": -405.38592529296875,
|
|
"logps/weighted_chosen": -3.4023194313049316,
|
|
"logps/weighted_rejected": -3.6703858375549316,
|
|
"loss": 0.5995,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -106.43359375,
|
|
"rewards/margins": 44.87871170043945,
|
|
"rewards/rejected": -151.3136749267578,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.7533203363418579,
|
|
"rewards/weighted_margins": 0.627819836139679,
|
|
"rewards/weighted_rejected": -1.3810546398162842,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.29835121695891126,
|
|
"grad_norm": 14.758890151977539,
|
|
"learning_rate": 8.85943037780415e-07,
|
|
"logits/chosen": -0.9987869262695312,
|
|
"logits/rejected": -1.0033996105194092,
|
|
"logps/chosen": -392.72344970703125,
|
|
"logps/rejected": -380.15625,
|
|
"logps/weighted_chosen": -3.2074217796325684,
|
|
"logps/weighted_rejected": -3.7137207984924316,
|
|
"loss": 0.6343,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -106.4300765991211,
|
|
"rewards/margins": 32.053321838378906,
|
|
"rewards/rejected": -138.525390625,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -0.744158923625946,
|
|
"rewards/weighted_margins": 0.547467052936554,
|
|
"rewards/weighted_rejected": -1.291876196861267,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.3035854488353834,
|
|
"grad_norm": 18.140382766723633,
|
|
"learning_rate": 8.800694230932884e-07,
|
|
"logits/chosen": -0.958880603313446,
|
|
"logits/rejected": -0.9720214605331421,
|
|
"logps/chosen": -390.57342529296875,
|
|
"logps/rejected": -398.5562438964844,
|
|
"logps/weighted_chosen": -2.86993408203125,
|
|
"logps/weighted_rejected": -3.4458250999450684,
|
|
"loss": 0.624,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": -100.91777038574219,
|
|
"rewards/margins": 32.43359375,
|
|
"rewards/rejected": -133.35546875,
|
|
"rewards/weighted_accuracies": 0.659375011920929,
|
|
"rewards/weighted_chosen": -0.558367908000946,
|
|
"rewards/weighted_margins": 0.4340454041957855,
|
|
"rewards/weighted_rejected": -0.992688000202179,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.30881968071185556,
|
|
"grad_norm": 16.685937881469727,
|
|
"learning_rate": 8.740688683062723e-07,
|
|
"logits/chosen": -1.0655395984649658,
|
|
"logits/rejected": -1.0949890613555908,
|
|
"logps/chosen": -449.0078125,
|
|
"logps/rejected": -423.9593811035156,
|
|
"logps/weighted_chosen": -2.912792921066284,
|
|
"logps/weighted_rejected": -3.656201124191284,
|
|
"loss": 0.6344,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": -119.9111328125,
|
|
"rewards/margins": 28.247461318969727,
|
|
"rewards/rejected": -148.1476593017578,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.5903121829032898,
|
|
"rewards/weighted_margins": 0.525067150592804,
|
|
"rewards/weighted_rejected": -1.1153564453125,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.31405391258832765,
|
|
"grad_norm": 20.500003814697266,
|
|
"learning_rate": 8.679433775559215e-07,
|
|
"logits/chosen": -1.0546143054962158,
|
|
"logits/rejected": -1.1019103527069092,
|
|
"logps/chosen": -473.6171875,
|
|
"logps/rejected": -477.95001220703125,
|
|
"logps/weighted_chosen": -2.896728515625,
|
|
"logps/weighted_rejected": -3.985156297683716,
|
|
"loss": 0.5741,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -148.9021453857422,
|
|
"rewards/margins": 43.50390625,
|
|
"rewards/rejected": -192.4656219482422,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -0.7574096918106079,
|
|
"rewards/weighted_margins": 0.587969958782196,
|
|
"rewards/weighted_rejected": -1.345495581626892,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.3192881444647998,
|
|
"grad_norm": 50.760562896728516,
|
|
"learning_rate": 8.616949967063871e-07,
|
|
"logits/chosen": -1.007971167564392,
|
|
"logits/rejected": -1.0581481456756592,
|
|
"logps/chosen": -419.1000061035156,
|
|
"logps/rejected": -460.9624938964844,
|
|
"logps/weighted_chosen": -3.3387207984924316,
|
|
"logps/weighted_rejected": -3.981640577316284,
|
|
"loss": 0.5971,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -150.87344360351562,
|
|
"rewards/margins": 49.451171875,
|
|
"rewards/rejected": -200.21444702148438,
|
|
"rewards/weighted_accuracies": 0.671875,
|
|
"rewards/weighted_chosen": -0.853253185749054,
|
|
"rewards/weighted_margins": 0.644561767578125,
|
|
"rewards/weighted_rejected": -1.497961401939392,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.3245223763412719,
|
|
"grad_norm": 16.369630813598633,
|
|
"learning_rate": 8.553258126661154e-07,
|
|
"logits/chosen": -1.0560119152069092,
|
|
"logits/rejected": -1.0674712657928467,
|
|
"logps/chosen": -442.5859375,
|
|
"logps/rejected": -463.5718688964844,
|
|
"logps/weighted_chosen": -3.429705858230591,
|
|
"logps/weighted_rejected": -4.259472846984863,
|
|
"loss": 0.6405,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -162.7121124267578,
|
|
"rewards/margins": 44.754295349121094,
|
|
"rewards/rejected": -207.43008422851562,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.9004287719726562,
|
|
"rewards/weighted_margins": 0.6113220453262329,
|
|
"rewards/weighted_rejected": -1.511694312095642,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.32975660821774405,
|
|
"grad_norm": 31.255151748657227,
|
|
"learning_rate": 8.488379526908368e-07,
|
|
"logits/chosen": -1.0301024913787842,
|
|
"logits/rejected": -1.0498230457305908,
|
|
"logps/chosen": -465.5859375,
|
|
"logps/rejected": -505.28125,
|
|
"logps/weighted_chosen": -3.3856444358825684,
|
|
"logps/weighted_rejected": -4.028759956359863,
|
|
"loss": 0.6384,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": -174.98046875,
|
|
"rewards/margins": 61.357032775878906,
|
|
"rewards/rejected": -236.38906860351562,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.969250500202179,
|
|
"rewards/weighted_margins": 0.5812011957168579,
|
|
"rewards/weighted_rejected": -1.5499999523162842,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.33499084009421615,
|
|
"grad_norm": 23.673486709594727,
|
|
"learning_rate": 8.422335836730802e-07,
|
|
"logits/chosen": -1.0135101079940796,
|
|
"logits/rejected": -1.0169677734375,
|
|
"logps/chosen": -426.2875061035156,
|
|
"logps/rejected": -497.64373779296875,
|
|
"logps/weighted_chosen": -3.098522901535034,
|
|
"logps/weighted_rejected": -3.898510694503784,
|
|
"loss": 0.6321,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -152.58749389648438,
|
|
"rewards/margins": 67.427734375,
|
|
"rewards/rejected": -220.0578155517578,
|
|
"rewards/weighted_accuracies": 0.699999988079071,
|
|
"rewards/weighted_chosen": -0.878277599811554,
|
|
"rewards/weighted_margins": 0.590301513671875,
|
|
"rewards/weighted_rejected": -1.468664526939392,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.3402250719706883,
|
|
"grad_norm": 20.44074821472168,
|
|
"learning_rate": 8.355149114184485e-07,
|
|
"logits/chosen": -1.105963110923767,
|
|
"logits/rejected": -1.095733642578125,
|
|
"logps/chosen": -474.7554626464844,
|
|
"logps/rejected": -523.0843505859375,
|
|
"logps/weighted_chosen": -3.1939697265625,
|
|
"logps/weighted_rejected": -3.786669969558716,
|
|
"loss": 0.5634,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -167.33828735351562,
|
|
"rewards/margins": 73.52656555175781,
|
|
"rewards/rejected": -240.85702514648438,
|
|
"rewards/weighted_accuracies": 0.668749988079071,
|
|
"rewards/weighted_chosen": -0.8172271847724915,
|
|
"rewards/weighted_margins": 0.6973236203193665,
|
|
"rewards/weighted_rejected": -1.514801025390625,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.34545930384716045,
|
|
"grad_norm": 25.958847045898438,
|
|
"learning_rate": 8.286841799088963e-07,
|
|
"logits/chosen": -1.1746642589569092,
|
|
"logits/rejected": -1.179473876953125,
|
|
"logps/chosen": -480.34063720703125,
|
|
"logps/rejected": -518.9437255859375,
|
|
"logps/weighted_chosen": -2.8995361328125,
|
|
"logps/weighted_rejected": -3.656787157058716,
|
|
"loss": 0.6073,
|
|
"rewards/accuracies": 0.5843750238418579,
|
|
"rewards/chosen": -196.3136749267578,
|
|
"rewards/margins": 59.513671875,
|
|
"rewards/rejected": -255.85897827148438,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -0.8131744265556335,
|
|
"rewards/weighted_margins": 0.576770007610321,
|
|
"rewards/weighted_rejected": -1.3900268077850342,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.35069353572363254,
|
|
"grad_norm": 15.336409568786621,
|
|
"learning_rate": 8.217436705532599e-07,
|
|
"logits/chosen": -1.1510436534881592,
|
|
"logits/rejected": -1.173730492591858,
|
|
"logps/chosen": -508.78594970703125,
|
|
"logps/rejected": -500.79217529296875,
|
|
"logps/weighted_chosen": -3.0011963844299316,
|
|
"logps/weighted_rejected": -3.7521729469299316,
|
|
"loss": 0.578,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -205.71914672851562,
|
|
"rewards/margins": 31.244531631469727,
|
|
"rewards/rejected": -237.14453125,
|
|
"rewards/weighted_accuracies": 0.721875011920929,
|
|
"rewards/weighted_chosen": -0.8121887445449829,
|
|
"rewards/weighted_margins": 0.688189685344696,
|
|
"rewards/weighted_rejected": -1.500115990638733,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.3559277676001047,
|
|
"grad_norm": 19.029943466186523,
|
|
"learning_rate": 8.14695701425284e-07,
|
|
"logits/chosen": -1.079345703125,
|
|
"logits/rejected": -1.125024437904358,
|
|
"logps/chosen": -466.38360595703125,
|
|
"logps/rejected": -470.4234313964844,
|
|
"logps/weighted_chosen": -2.9954833984375,
|
|
"logps/weighted_rejected": -3.8232421875,
|
|
"loss": 0.5882,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -160.8718719482422,
|
|
"rewards/margins": 47.43242263793945,
|
|
"rewards/rejected": -208.275390625,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.7394653558731079,
|
|
"rewards/weighted_margins": 0.627307116985321,
|
|
"rewards/weighted_rejected": -1.3663330078125,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.3611619994765768,
|
|
"grad_norm": 13.842121124267578,
|
|
"learning_rate": 8.075426264894046e-07,
|
|
"logits/chosen": -1.013006567955017,
|
|
"logits/rejected": -1.0405089855194092,
|
|
"logps/chosen": -472.03436279296875,
|
|
"logps/rejected": -501.09375,
|
|
"logps/weighted_chosen": -3.1174073219299316,
|
|
"logps/weighted_rejected": -4.429980278015137,
|
|
"loss": 0.5163,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -166.5642547607422,
|
|
"rewards/margins": 56.965431213378906,
|
|
"rewards/rejected": -223.5593719482422,
|
|
"rewards/weighted_accuracies": 0.778124988079071,
|
|
"rewards/weighted_chosen": -0.7782195806503296,
|
|
"rewards/weighted_margins": 0.887194812297821,
|
|
"rewards/weighted_rejected": -1.665307641029358,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.36639623135304894,
|
|
"grad_norm": 31.751785278320312,
|
|
"learning_rate": 8.002868348145435e-07,
|
|
"logits/chosen": -1.0329437255859375,
|
|
"logits/rejected": -1.0401580333709717,
|
|
"logps/chosen": -535.6437377929688,
|
|
"logps/rejected": -564.8609619140625,
|
|
"logps/weighted_chosen": -3.4560546875,
|
|
"logps/weighted_rejected": -3.8817505836486816,
|
|
"loss": 0.5956,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -235.3767547607422,
|
|
"rewards/margins": 61.516014099121094,
|
|
"rewards/rejected": -296.88360595703125,
|
|
"rewards/weighted_accuracies": 0.71875,
|
|
"rewards/weighted_chosen": -1.1499541997909546,
|
|
"rewards/weighted_margins": 0.6718994379043579,
|
|
"rewards/weighted_rejected": -1.8215301036834717,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.3716304632295211,
|
|
"grad_norm": 32.05786895751953,
|
|
"learning_rate": 7.92930749776179e-07,
|
|
"logits/chosen": -1.0492126941680908,
|
|
"logits/rejected": -1.092248558998108,
|
|
"logps/chosen": -564.2515869140625,
|
|
"logps/rejected": -602.4500122070312,
|
|
"logps/weighted_chosen": -3.8183350563049316,
|
|
"logps/weighted_rejected": -4.569482326507568,
|
|
"loss": 0.652,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -292.5419921875,
|
|
"rewards/margins": 49.53203201293945,
|
|
"rewards/rejected": -342.0179748535156,
|
|
"rewards/weighted_accuracies": 0.6656249761581421,
|
|
"rewards/weighted_chosen": -1.244970679283142,
|
|
"rewards/weighted_margins": 0.630566418170929,
|
|
"rewards/weighted_rejected": -1.8747985363006592,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.3768646951059932,
|
|
"grad_norm": 23.8446044921875,
|
|
"learning_rate": 7.854768282469582e-07,
|
|
"logits/chosen": -1.0707213878631592,
|
|
"logits/rejected": -1.126983642578125,
|
|
"logps/chosen": -462.90155029296875,
|
|
"logps/rejected": -542.5218505859375,
|
|
"logps/weighted_chosen": -3.208447217941284,
|
|
"logps/weighted_rejected": -4.043383598327637,
|
|
"loss": 0.6115,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -185.4597625732422,
|
|
"rewards/margins": 79.3746109008789,
|
|
"rewards/rejected": -264.8812561035156,
|
|
"rewards/weighted_accuracies": 0.6781250238418579,
|
|
"rewards/weighted_chosen": -0.8762931823730469,
|
|
"rewards/weighted_margins": 0.711090087890625,
|
|
"rewards/weighted_rejected": -1.5873870849609375,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.38209892698246534,
|
|
"grad_norm": 17.816680908203125,
|
|
"learning_rate": 7.779275597761215e-07,
|
|
"logits/chosen": -1.036431908607483,
|
|
"logits/rejected": -1.105950951576233,
|
|
"logps/chosen": -471.43280029296875,
|
|
"logps/rejected": -532.6843872070312,
|
|
"logps/weighted_chosen": -3.378674268722534,
|
|
"logps/weighted_rejected": -4.096581935882568,
|
|
"loss": 0.5442,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -195.43984985351562,
|
|
"rewards/margins": 68.92460632324219,
|
|
"rewards/rejected": -264.296875,
|
|
"rewards/weighted_accuracies": 0.706250011920929,
|
|
"rewards/weighted_chosen": -0.847393810749054,
|
|
"rewards/weighted_margins": 0.777783215045929,
|
|
"rewards/weighted_rejected": -1.625451683998108,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.38733315885893743,
|
|
"grad_norm": 29.750192642211914,
|
|
"learning_rate": 7.702854657580126e-07,
|
|
"logits/chosen": -1.0928223133087158,
|
|
"logits/rejected": -1.1139404773712158,
|
|
"logps/chosen": -507.796875,
|
|
"logps/rejected": -523.7952880859375,
|
|
"logps/weighted_chosen": -3.3760986328125,
|
|
"logps/weighted_rejected": -4.447509765625,
|
|
"loss": 0.5597,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -213.2277374267578,
|
|
"rewards/margins": 56.09648513793945,
|
|
"rewards/rejected": -269.4203186035156,
|
|
"rewards/weighted_accuracies": 0.7406250238418579,
|
|
"rewards/weighted_chosen": -0.927258312702179,
|
|
"rewards/weighted_margins": 0.8479980230331421,
|
|
"rewards/weighted_rejected": -1.7749145030975342,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.3925673907354096,
|
|
"grad_norm": 38.97298812866211,
|
|
"learning_rate": 7.625530985899547e-07,
|
|
"logits/chosen": -1.0656921863555908,
|
|
"logits/rejected": -1.0953140258789062,
|
|
"logps/chosen": -491.4609375,
|
|
"logps/rejected": -545.0203247070312,
|
|
"logps/weighted_chosen": -3.2592406272888184,
|
|
"logps/weighted_rejected": -4.605615139007568,
|
|
"loss": 0.5776,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -224.6374969482422,
|
|
"rewards/margins": 75.23710632324219,
|
|
"rewards/rejected": -300.0316467285156,
|
|
"rewards/weighted_accuracies": 0.699999988079071,
|
|
"rewards/weighted_chosen": -1.093621850013733,
|
|
"rewards/weighted_margins": 0.852429211139679,
|
|
"rewards/weighted_rejected": -1.9456908702850342,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.39780162261188173,
|
|
"grad_norm": 22.909940719604492,
|
|
"learning_rate": 7.547330408197694e-07,
|
|
"logits/chosen": -1.0663268566131592,
|
|
"logits/rejected": -1.1003234386444092,
|
|
"logps/chosen": -572.404296875,
|
|
"logps/rejected": -556.4031372070312,
|
|
"logps/weighted_chosen": -3.372363328933716,
|
|
"logps/weighted_rejected": -4.400781154632568,
|
|
"loss": 0.5943,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": -276.7529296875,
|
|
"rewards/margins": 23.257421493530273,
|
|
"rewards/rejected": -300.1558532714844,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -1.1045868396759033,
|
|
"rewards/weighted_margins": 0.7262054681777954,
|
|
"rewards/weighted_rejected": -1.8311798572540283,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.40303585448835383,
|
|
"grad_norm": 107.9198989868164,
|
|
"learning_rate": 7.468279042832271e-07,
|
|
"logits/chosen": -1.067718505859375,
|
|
"logits/rejected": -1.095727562904358,
|
|
"logps/chosen": -467.5,
|
|
"logps/rejected": -559.7703247070312,
|
|
"logps/weighted_chosen": -3.0786986351013184,
|
|
"logps/weighted_rejected": -3.8573241233825684,
|
|
"loss": 0.5995,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -187.9324188232422,
|
|
"rewards/margins": 82.28340148925781,
|
|
"rewards/rejected": -270.2611389160156,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -0.867828369140625,
|
|
"rewards/weighted_margins": 0.625274658203125,
|
|
"rewards/weighted_rejected": -1.493688941001892,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.408270086364826,
|
|
"grad_norm": 18.453107833862305,
|
|
"learning_rate": 7.388403292317154e-07,
|
|
"logits/chosen": -1.0655944347381592,
|
|
"logits/rejected": -1.121740698814392,
|
|
"logps/chosen": -503.8828125,
|
|
"logps/rejected": -519.8375244140625,
|
|
"logps/weighted_chosen": -3.039843797683716,
|
|
"logps/weighted_rejected": -3.848388671875,
|
|
"loss": 0.5822,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -202.9210968017578,
|
|
"rewards/margins": 50.95390701293945,
|
|
"rewards/rejected": -253.869140625,
|
|
"rewards/weighted_accuracies": 0.731249988079071,
|
|
"rewards/weighted_chosen": -0.873791515827179,
|
|
"rewards/weighted_margins": 0.6851806640625,
|
|
"rewards/weighted_rejected": -1.5595581531524658,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.4135043182412981,
|
|
"grad_norm": 24.38869285583496,
|
|
"learning_rate": 7.307729834504154e-07,
|
|
"logits/chosen": -1.090570092201233,
|
|
"logits/rejected": -1.1371276378631592,
|
|
"logps/chosen": -490.86407470703125,
|
|
"logps/rejected": -537.7296752929688,
|
|
"logps/weighted_chosen": -3.041918992996216,
|
|
"logps/weighted_rejected": -4.070898532867432,
|
|
"loss": 0.6125,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -209.13241577148438,
|
|
"rewards/margins": 62.44609451293945,
|
|
"rewards/rejected": -271.58203125,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -0.947405993938446,
|
|
"rewards/weighted_margins": 0.609375,
|
|
"rewards/weighted_rejected": -1.556726098060608,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.4187385501177702,
|
|
"grad_norm": 26.962316513061523,
|
|
"learning_rate": 7.226285613672847e-07,
|
|
"logits/chosen": -1.03680419921875,
|
|
"logits/rejected": -1.0742919445037842,
|
|
"logps/chosen": -500.84295654296875,
|
|
"logps/rejected": -603.6422119140625,
|
|
"logps/weighted_chosen": -3.154052734375,
|
|
"logps/weighted_rejected": -4.342724800109863,
|
|
"loss": 0.5564,
|
|
"rewards/accuracies": 0.671875,
|
|
"rewards/chosen": -223.51171875,
|
|
"rewards/margins": 101.2464828491211,
|
|
"rewards/rejected": -324.720703125,
|
|
"rewards/weighted_accuracies": 0.7406250238418579,
|
|
"rewards/weighted_chosen": -0.996997058391571,
|
|
"rewards/weighted_margins": 0.755480945110321,
|
|
"rewards/weighted_rejected": -1.752661108970642,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.4239727819942423,
|
|
"grad_norm": 15.780332565307617,
|
|
"learning_rate": 7.144097831531398e-07,
|
|
"logits/chosen": -1.0250060558319092,
|
|
"logits/rejected": -1.052709937095642,
|
|
"logps/chosen": -517.7921752929688,
|
|
"logps/rejected": -600.5421752929688,
|
|
"logps/weighted_chosen": -3.318652391433716,
|
|
"logps/weighted_rejected": -4.39453125,
|
|
"loss": 0.5695,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -247.40664672851562,
|
|
"rewards/margins": 88.87187194824219,
|
|
"rewards/rejected": -336.234375,
|
|
"rewards/weighted_accuracies": 0.703125,
|
|
"rewards/weighted_chosen": -1.082605004310608,
|
|
"rewards/weighted_margins": 0.89263916015625,
|
|
"rewards/weighted_rejected": -1.974450707435608,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.42920701387071447,
|
|
"grad_norm": 24.66672706604004,
|
|
"learning_rate": 7.061193938131396e-07,
|
|
"logits/chosen": -0.9916015863418579,
|
|
"logits/rejected": -1.042016625404358,
|
|
"logps/chosen": -607.3375244140625,
|
|
"logps/rejected": -642.5671997070312,
|
|
"logps/weighted_chosen": -3.689379930496216,
|
|
"logps/weighted_rejected": -4.398633003234863,
|
|
"loss": 0.5654,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -307.7925720214844,
|
|
"rewards/margins": 69.5777359008789,
|
|
"rewards/rejected": -377.3050842285156,
|
|
"rewards/weighted_accuracies": 0.7124999761581421,
|
|
"rewards/weighted_chosen": -1.1716430187225342,
|
|
"rewards/weighted_margins": 0.9023987054824829,
|
|
"rewards/weighted_rejected": -2.074475049972534,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.4344412457471866,
|
|
"grad_norm": 20.459196090698242,
|
|
"learning_rate": 6.977601622699789e-07,
|
|
"logits/chosen": -0.9984985589981079,
|
|
"logits/rejected": -1.079248070716858,
|
|
"logps/chosen": -471.5625,
|
|
"logps/rejected": -574.8695068359375,
|
|
"logps/weighted_chosen": -3.233837842941284,
|
|
"logps/weighted_rejected": -4.211133003234863,
|
|
"loss": 0.5097,
|
|
"rewards/accuracies": 0.6656249761581421,
|
|
"rewards/chosen": -186.7394561767578,
|
|
"rewards/margins": 109.28633117675781,
|
|
"rewards/rejected": -296.13555908203125,
|
|
"rewards/weighted_accuracies": 0.746874988079071,
|
|
"rewards/weighted_chosen": -0.760040283203125,
|
|
"rewards/weighted_margins": 0.9420166015625,
|
|
"rewards/weighted_rejected": -1.7019164562225342,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.4396754776236587,
|
|
"grad_norm": 89.301025390625,
|
|
"learning_rate": 6.893348804390882e-07,
|
|
"logits/chosen": -1.126031517982483,
|
|
"logits/rejected": -1.1464111804962158,
|
|
"logps/chosen": -541.5546875,
|
|
"logps/rejected": -580.4109497070312,
|
|
"logps/weighted_chosen": -3.6358885765075684,
|
|
"logps/weighted_rejected": -4.262304782867432,
|
|
"loss": 0.566,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -246.6085968017578,
|
|
"rewards/margins": 74.0562515258789,
|
|
"rewards/rejected": -320.61328125,
|
|
"rewards/weighted_accuracies": 0.684374988079071,
|
|
"rewards/weighted_chosen": -1.0077393054962158,
|
|
"rewards/weighted_margins": 0.7990967035293579,
|
|
"rewards/weighted_rejected": -1.806396484375,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.44490970950013087,
|
|
"grad_norm": 20.6274471282959,
|
|
"learning_rate": 6.808463622961578e-07,
|
|
"logits/chosen": -1.151123046875,
|
|
"logits/rejected": -1.2046630382537842,
|
|
"logps/chosen": -621.8390502929688,
|
|
"logps/rejected": -716.7249755859375,
|
|
"logps/weighted_chosen": -3.6482911109924316,
|
|
"logps/weighted_rejected": -4.6015625,
|
|
"loss": 0.5759,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -328.931640625,
|
|
"rewards/margins": 106.0474624633789,
|
|
"rewards/rejected": -435.07403564453125,
|
|
"rewards/weighted_accuracies": 0.7124999761581421,
|
|
"rewards/weighted_chosen": -1.2104400396347046,
|
|
"rewards/weighted_margins": 0.9383910894393921,
|
|
"rewards/weighted_rejected": -2.1482911109924316,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.45014394137660296,
|
|
"grad_norm": 20.91545295715332,
|
|
"learning_rate": 6.722974429372925e-07,
|
|
"logits/chosen": -1.1297729015350342,
|
|
"logits/rejected": -1.170617699623108,
|
|
"logps/chosen": -697.5484619140625,
|
|
"logps/rejected": -730.65625,
|
|
"logps/weighted_chosen": -3.5313477516174316,
|
|
"logps/weighted_rejected": -5.184765815734863,
|
|
"loss": 0.5217,
|
|
"rewards/accuracies": 0.5718749761581421,
|
|
"rewards/chosen": -393.822265625,
|
|
"rewards/margins": 74.92500305175781,
|
|
"rewards/rejected": -468.8492126464844,
|
|
"rewards/weighted_accuracies": 0.7718750238418579,
|
|
"rewards/weighted_chosen": -1.393365502357483,
|
|
"rewards/weighted_margins": 1.1208374500274658,
|
|
"rewards/weighted_rejected": -2.514172315597534,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.4553781732530751,
|
|
"grad_norm": 42.88610076904297,
|
|
"learning_rate": 6.636909776321128e-07,
|
|
"logits/chosen": -1.1412475109100342,
|
|
"logits/rejected": -1.1436645984649658,
|
|
"logps/chosen": -549.9671630859375,
|
|
"logps/rejected": -677.8781127929688,
|
|
"logps/weighted_chosen": -3.683642625808716,
|
|
"logps/weighted_rejected": -4.7724609375,
|
|
"loss": 0.5195,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -288.21484375,
|
|
"rewards/margins": 124.931640625,
|
|
"rewards/rejected": -413.17266845703125,
|
|
"rewards/weighted_accuracies": 0.7250000238418579,
|
|
"rewards/weighted_chosen": -1.2497345209121704,
|
|
"rewards/weighted_margins": 0.971630871295929,
|
|
"rewards/weighted_rejected": -2.2208008766174316,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.46061240512954726,
|
|
"grad_norm": 29.938108444213867,
|
|
"learning_rate": 6.550298408701174e-07,
|
|
"logits/chosen": -1.0770995616912842,
|
|
"logits/rejected": -1.134680151939392,
|
|
"logps/chosen": -574.3234252929688,
|
|
"logps/rejected": -675.2101440429688,
|
|
"logps/weighted_chosen": -3.807568311691284,
|
|
"logps/weighted_rejected": -5.117138862609863,
|
|
"loss": 0.5413,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -287.63494873046875,
|
|
"rewards/margins": 104.92851257324219,
|
|
"rewards/rejected": -392.5396423339844,
|
|
"rewards/weighted_accuracies": 0.734375,
|
|
"rewards/weighted_chosen": -1.1316711902618408,
|
|
"rewards/weighted_margins": 0.954119861125946,
|
|
"rewards/weighted_rejected": -2.0862364768981934,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.46584663700601936,
|
|
"grad_norm": 27.375259399414062,
|
|
"learning_rate": 6.463169254006276e-07,
|
|
"logits/chosen": -1.10406494140625,
|
|
"logits/rejected": -1.156030297279358,
|
|
"logps/chosen": -541.8601684570312,
|
|
"logps/rejected": -578.5452880859375,
|
|
"logps/weighted_chosen": -3.4522461891174316,
|
|
"logps/weighted_rejected": -4.527783393859863,
|
|
"loss": 0.5321,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -260.65704345703125,
|
|
"rewards/margins": 71.1494140625,
|
|
"rewards/rejected": -331.875,
|
|
"rewards/weighted_accuracies": 0.7281249761581421,
|
|
"rewards/weighted_chosen": -0.977038562297821,
|
|
"rewards/weighted_margins": 1.022253394126892,
|
|
"rewards/weighted_rejected": -1.999292016029358,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.4710808688824915,
|
|
"grad_norm": 32.590675354003906,
|
|
"learning_rate": 6.375551412666326e-07,
|
|
"logits/chosen": -1.0722472667694092,
|
|
"logits/rejected": -1.1129913330078125,
|
|
"logps/chosen": -499.8031311035156,
|
|
"logps/rejected": -543.9796752929688,
|
|
"logps/weighted_chosen": -3.171948194503784,
|
|
"logps/weighted_rejected": -4.273193359375,
|
|
"loss": 0.6035,
|
|
"rewards/accuracies": 0.590624988079071,
|
|
"rewards/chosen": -225.7996063232422,
|
|
"rewards/margins": 61.669532775878906,
|
|
"rewards/rejected": -287.5015563964844,
|
|
"rewards/weighted_accuracies": 0.6812499761581421,
|
|
"rewards/weighted_chosen": -1.0042235851287842,
|
|
"rewards/weighted_margins": 0.7809082269668579,
|
|
"rewards/weighted_rejected": -1.784277319908142,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.4763151007589636,
|
|
"grad_norm": 37.88996887207031,
|
|
"learning_rate": 6.287474148328583e-07,
|
|
"logits/chosen": -1.0318695306777954,
|
|
"logits/rejected": -1.0187164545059204,
|
|
"logps/chosen": -491.97186279296875,
|
|
"logps/rejected": -540.4202880859375,
|
|
"logps/weighted_chosen": -3.5292601585388184,
|
|
"logps/weighted_rejected": -5.02734375,
|
|
"loss": 0.559,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -221.07968139648438,
|
|
"rewards/margins": 72.57441711425781,
|
|
"rewards/rejected": -293.56561279296875,
|
|
"rewards/weighted_accuracies": 0.690625011920929,
|
|
"rewards/weighted_chosen": -1.0068145990371704,
|
|
"rewards/weighted_margins": 0.84521484375,
|
|
"rewards/weighted_rejected": -1.851770043373108,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.48154933263543576,
|
|
"grad_norm": 25.340351104736328,
|
|
"learning_rate": 6.198966878083857e-07,
|
|
"logits/chosen": -1.0930664539337158,
|
|
"logits/rejected": -1.12109375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -623.390625,
|
|
"logps/weighted_chosen": -3.68231201171875,
|
|
"logps/weighted_rejected": -4.464990139007568,
|
|
"loss": 0.5911,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -268.6884765625,
|
|
"rewards/margins": 87.7593765258789,
|
|
"rewards/rejected": -356.4453125,
|
|
"rewards/weighted_accuracies": 0.7124999761581421,
|
|
"rewards/weighted_chosen": -1.085455298423767,
|
|
"rewards/weighted_margins": 0.79486083984375,
|
|
"rewards/weighted_rejected": -1.880859375,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.48678356451190785,
|
|
"grad_norm": 47.73952865600586,
|
|
"learning_rate": 6.110059162641439e-07,
|
|
"logits/chosen": -1.1020386219024658,
|
|
"logits/rejected": -1.1357238292694092,
|
|
"logps/chosen": -533.7390747070312,
|
|
"logps/rejected": -599.3781127929688,
|
|
"logps/weighted_chosen": -2.9868531227111816,
|
|
"logps/weighted_rejected": -4.054858207702637,
|
|
"loss": 0.5458,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -251.13125610351562,
|
|
"rewards/margins": 80.9544906616211,
|
|
"rewards/rejected": -332.2699279785156,
|
|
"rewards/weighted_accuracies": 0.734375,
|
|
"rewards/weighted_chosen": -0.8615478277206421,
|
|
"rewards/weighted_margins": 0.8154662847518921,
|
|
"rewards/weighted_rejected": -1.6761963367462158,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.49201779638838,
|
|
"grad_norm": 36.5290412902832,
|
|
"learning_rate": 6.020780696456059e-07,
|
|
"logits/chosen": -1.133764624595642,
|
|
"logits/rejected": -1.177819848060608,
|
|
"logps/chosen": -515.2952880859375,
|
|
"logps/rejected": -611.4953002929688,
|
|
"logps/weighted_chosen": -3.0075440406799316,
|
|
"logps/weighted_rejected": -4.250219821929932,
|
|
"loss": 0.5361,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -244.7570343017578,
|
|
"rewards/margins": 99.73710632324219,
|
|
"rewards/rejected": -344.3531188964844,
|
|
"rewards/weighted_accuracies": 0.731249988079071,
|
|
"rewards/weighted_chosen": -1.0392029285430908,
|
|
"rewards/weighted_margins": 0.8944336175918579,
|
|
"rewards/weighted_rejected": -1.93341064453125,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.49725202826485215,
|
|
"grad_norm": 36.72229766845703,
|
|
"learning_rate": 5.931161297810185e-07,
|
|
"logits/chosen": -1.2070038318634033,
|
|
"logits/rejected": -1.23931884765625,
|
|
"logps/chosen": -550.4031372070312,
|
|
"logps/rejected": -615.3375244140625,
|
|
"logps/weighted_chosen": -3.748730421066284,
|
|
"logps/weighted_rejected": -4.64697265625,
|
|
"loss": 0.5726,
|
|
"rewards/accuracies": 0.5843750238418579,
|
|
"rewards/chosen": -277.07733154296875,
|
|
"rewards/margins": 77.06562805175781,
|
|
"rewards/rejected": -354.22381591796875,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -1.09320068359375,
|
|
"rewards/weighted_margins": 0.812207043170929,
|
|
"rewards/weighted_rejected": -1.906347632408142,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.5024862601413242,
|
|
"grad_norm": 39.58312225341797,
|
|
"learning_rate": 5.841230898854959e-07,
|
|
"logits/chosen": -1.1735107898712158,
|
|
"logits/rejected": -1.2161743640899658,
|
|
"logps/chosen": -684.3687744140625,
|
|
"logps/rejected": -794.3922119140625,
|
|
"logps/weighted_chosen": -3.8369140625,
|
|
"logps/weighted_rejected": -4.986083984375,
|
|
"loss": 0.6144,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -384.17852783203125,
|
|
"rewards/margins": 148.6707000732422,
|
|
"rewards/rejected": -532.8191528320312,
|
|
"rewards/weighted_accuracies": 0.6937500238418579,
|
|
"rewards/weighted_chosen": -1.5656402111053467,
|
|
"rewards/weighted_margins": 0.9439452886581421,
|
|
"rewards/weighted_rejected": -2.5091795921325684,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.5077204920177963,
|
|
"grad_norm": 24.286752700805664,
|
|
"learning_rate": 5.751019535613102e-07,
|
|
"logits/chosen": -1.0753357410430908,
|
|
"logits/rejected": -1.104547142982483,
|
|
"logps/chosen": -550.9773559570312,
|
|
"logps/rejected": -653.6687622070312,
|
|
"logps/weighted_chosen": -3.5946044921875,
|
|
"logps/weighted_rejected": -5.038866996765137,
|
|
"loss": 0.5266,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -291.25390625,
|
|
"rewards/margins": 115.4117202758789,
|
|
"rewards/rejected": -406.62811279296875,
|
|
"rewards/weighted_accuracies": 0.7093750238418579,
|
|
"rewards/weighted_chosen": -1.2581603527069092,
|
|
"rewards/weighted_margins": 1.127111792564392,
|
|
"rewards/weighted_rejected": -2.3849730491638184,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.5129547238942685,
|
|
"grad_norm": 22.55328941345215,
|
|
"learning_rate": 5.660557337947117e-07,
|
|
"logits/chosen": -1.0788085460662842,
|
|
"logits/rejected": -1.109989881515503,
|
|
"logps/chosen": -594.8406372070312,
|
|
"logps/rejected": -653.6624755859375,
|
|
"logps/weighted_chosen": -3.10791015625,
|
|
"logps/weighted_rejected": -4.297143459320068,
|
|
"loss": 0.5361,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -300.37872314453125,
|
|
"rewards/margins": 95.828125,
|
|
"rewards/rejected": -396.203125,
|
|
"rewards/weighted_accuracies": 0.6968749761581421,
|
|
"rewards/weighted_chosen": -1.0415496826171875,
|
|
"rewards/weighted_margins": 0.88812255859375,
|
|
"rewards/weighted_rejected": -1.929406762123108,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.5181889557707406,
|
|
"grad_norm": 53.25763702392578,
|
|
"learning_rate": 5.569874519496174e-07,
|
|
"logits/chosen": -1.055578589439392,
|
|
"logits/rejected": -1.122521996498108,
|
|
"logps/chosen": -497.3999938964844,
|
|
"logps/rejected": -579.3656005859375,
|
|
"logps/weighted_chosen": -3.393261671066284,
|
|
"logps/weighted_rejected": -4.673925876617432,
|
|
"loss": 0.5302,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -222.11483764648438,
|
|
"rewards/margins": 90.296875,
|
|
"rewards/rejected": -312.42108154296875,
|
|
"rewards/weighted_accuracies": 0.699999988079071,
|
|
"rewards/weighted_chosen": -0.8590606451034546,
|
|
"rewards/weighted_margins": 0.879437267780304,
|
|
"rewards/weighted_rejected": -1.7384154796600342,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"grad_norm": 39.35697555541992,
|
|
"learning_rate": 5.47900136758499e-07,
|
|
"logits/chosen": -1.0391693115234375,
|
|
"logits/rejected": -1.1183960437774658,
|
|
"logps/chosen": -589.5906372070312,
|
|
"logps/rejected": -653.09375,
|
|
"logps/weighted_chosen": -3.634228467941284,
|
|
"logps/weighted_rejected": -4.7723388671875,
|
|
"loss": 0.5255,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": -321.95623779296875,
|
|
"rewards/margins": 89.08320617675781,
|
|
"rewards/rejected": -410.8277282714844,
|
|
"rewards/weighted_accuracies": 0.731249988079071,
|
|
"rewards/weighted_chosen": -1.187219262123108,
|
|
"rewards/weighted_margins": 0.9879394769668579,
|
|
"rewards/weighted_rejected": -2.1756300926208496,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"eval_logits/chosen": -1.194671869277954,
|
|
"eval_logits/rejected": -1.2200820446014404,
|
|
"eval_logps/chosen": -645.4445190429688,
|
|
"eval_logps/rejected": -726.552001953125,
|
|
"eval_logps/weighted_chosen": -3.7153730392456055,
|
|
"eval_logps/weighted_rejected": -4.821405410766602,
|
|
"eval_loss": 0.5563206672668457,
|
|
"eval_rewards/accuracies": 0.5899999737739563,
|
|
"eval_rewards/chosen": -356.71826171875,
|
|
"eval_rewards/margins": 102.17925262451172,
|
|
"eval_rewards/rejected": -458.8554992675781,
|
|
"eval_rewards/weighted_accuracies": 0.7135000228881836,
|
|
"eval_rewards/weighted_chosen": -1.3285291194915771,
|
|
"eval_rewards/weighted_margins": 0.9376659989356995,
|
|
"eval_rewards/weighted_rejected": -2.266195297241211,
|
|
"eval_runtime": 1365.9873,
|
|
"eval_samples_per_second": 1.464,
|
|
"eval_steps_per_second": 0.366,
|
|
"step": 1000
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1911,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|