40380 lines
1.2 MiB
40380 lines
1.2 MiB
{
|
|
"best_metric": 0.4581940472126007,
|
|
"best_model_checkpoint": "models/qwen2.5-3b-dpo-vanilla/checkpoint-10000",
|
|
"epoch": 0.9999813887699838,
|
|
"eval_steps": 10000,
|
|
"global_step": 26865,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 3.722246003238354e-05,
|
|
"grad_norm": 1.909734052867258,
|
|
"learning_rate": 1.8608113137327875e-10,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -128.0,
|
|
"logps/rejected": -128.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0003722246003238354,
|
|
"grad_norm": 2.27001634205206,
|
|
"learning_rate": 1.8608113137327875e-09,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6921,
|
|
"rewards/accuracies": 0.0902777761220932,
|
|
"rewards/chosen": -0.0004520416259765625,
|
|
"rewards/margins": -0.00093841552734375,
|
|
"rewards/rejected": 0.0004863739013671875,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0007444492006476708,
|
|
"grad_norm": 1.944229602517856,
|
|
"learning_rate": 3.721622627465575e-09,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -196.0,
|
|
"logps/rejected": -175.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3187499940395355,
|
|
"rewards/chosen": -0.000640869140625,
|
|
"rewards/margins": -0.0001888275146484375,
|
|
"rewards/rejected": -0.000453948974609375,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0011166738009715062,
|
|
"grad_norm": 1.6686587196870428,
|
|
"learning_rate": 5.5824339411983625e-09,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -183.0,
|
|
"logps/rejected": -164.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.30000001192092896,
|
|
"rewards/chosen": -0.00086212158203125,
|
|
"rewards/margins": -0.0009918212890625,
|
|
"rewards/rejected": 0.00012493133544921875,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.0014888984012953416,
|
|
"grad_norm": 1.6479666303692577,
|
|
"learning_rate": 7.44324525493115e-09,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": -1.2993812561035156e-05,
|
|
"rewards/margins": 0.0007781982421875,
|
|
"rewards/rejected": -0.000797271728515625,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.001861123001619177,
|
|
"grad_norm": 1.8722931582272013,
|
|
"learning_rate": 9.304056568663937e-09,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -177.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.30000001192092896,
|
|
"rewards/chosen": 0.00015354156494140625,
|
|
"rewards/margins": 0.000186920166015625,
|
|
"rewards/rejected": -2.9802322387695312e-05,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.0022333476019430125,
|
|
"grad_norm": 1.8791508809725042,
|
|
"learning_rate": 1.1164867882396725e-08,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -170.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.33125001192092896,
|
|
"rewards/chosen": 0.0006103515625,
|
|
"rewards/margins": 0.00054931640625,
|
|
"rewards/rejected": 6.246566772460938e-05,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.002605572202266848,
|
|
"grad_norm": 2.026846571369839,
|
|
"learning_rate": 1.3025679196129512e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3062500059604645,
|
|
"rewards/chosen": -0.0004405975341796875,
|
|
"rewards/margins": 0.00020503997802734375,
|
|
"rewards/rejected": -0.000640869140625,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.002977796802590683,
|
|
"grad_norm": 1.8717258776080847,
|
|
"learning_rate": 1.48864905098623e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -161.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": -0.000766754150390625,
|
|
"rewards/margins": 0.00067138671875,
|
|
"rewards/rejected": -0.00144195556640625,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.0033500214029145185,
|
|
"grad_norm": 1.829868682345127,
|
|
"learning_rate": 1.6747301823595087e-08,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -196.0,
|
|
"logps/rejected": -186.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": -0.00018596649169921875,
|
|
"rewards/margins": -0.00018596649169921875,
|
|
"rewards/rejected": -7.636845111846924e-07,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.003722246003238354,
|
|
"grad_norm": 1.9924116659754334,
|
|
"learning_rate": 1.8608113137327873e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -211.0,
|
|
"logps/rejected": -179.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 0.00140380859375,
|
|
"rewards/margins": 0.002227783203125,
|
|
"rewards/rejected": -0.000812530517578125,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.004094470603562189,
|
|
"grad_norm": 1.7583918647858459,
|
|
"learning_rate": 2.0468924451060663e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -196.0,
|
|
"logps/rejected": -176.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": -0.000202178955078125,
|
|
"rewards/margins": -0.000156402587890625,
|
|
"rewards/rejected": -4.649162292480469e-05,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.004466695203886025,
|
|
"grad_norm": 1.749356736425924,
|
|
"learning_rate": 2.232973576479345e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -182.0,
|
|
"logps/rejected": -173.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": -0.001190185546875,
|
|
"rewards/margins": -0.000263214111328125,
|
|
"rewards/rejected": -0.00092315673828125,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.00483891980420986,
|
|
"grad_norm": 1.9863481162171532,
|
|
"learning_rate": 2.4190547078526237e-08,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -180.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3062500059604645,
|
|
"rewards/chosen": -0.00138092041015625,
|
|
"rewards/margins": 0.000141143798828125,
|
|
"rewards/rejected": -0.00151824951171875,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.005211144404533696,
|
|
"grad_norm": 1.7434811793002054,
|
|
"learning_rate": 2.6051358392259023e-08,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": -0.000797271728515625,
|
|
"rewards/margins": 0.0004863739013671875,
|
|
"rewards/rejected": -0.00128173828125,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.005583369004857531,
|
|
"grad_norm": 2.0956524520987228,
|
|
"learning_rate": 2.791216970599181e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.00041961669921875,
|
|
"rewards/margins": 0.0001392364501953125,
|
|
"rewards/rejected": 0.00028228759765625,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.005955593605181366,
|
|
"grad_norm": 1.956063700802031,
|
|
"learning_rate": 2.97729810197246e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -176.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": 0.0004367828369140625,
|
|
"rewards/margins": -0.0003910064697265625,
|
|
"rewards/rejected": 0.00083160400390625,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.006327818205505202,
|
|
"grad_norm": 1.8958883076227333,
|
|
"learning_rate": 3.163379233345739e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": 0.00084686279296875,
|
|
"rewards/margins": 0.001068115234375,
|
|
"rewards/rejected": -0.00021839141845703125,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.006700042805829037,
|
|
"grad_norm": 1.7428685916939655,
|
|
"learning_rate": 3.3494603647190173e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -189.0,
|
|
"logps/rejected": -159.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.38749998807907104,
|
|
"rewards/chosen": -0.000423431396484375,
|
|
"rewards/margins": 0.0003910064697265625,
|
|
"rewards/rejected": -0.00081634521484375,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.007072267406152873,
|
|
"grad_norm": 1.9433027302325678,
|
|
"learning_rate": 3.535541496092296e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.32499998807907104,
|
|
"rewards/chosen": -0.000438690185546875,
|
|
"rewards/margins": -0.00102996826171875,
|
|
"rewards/rejected": 0.0005950927734375,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.007444492006476708,
|
|
"grad_norm": 1.907942453475148,
|
|
"learning_rate": 3.721622627465575e-08,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -182.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": 0.0013275146484375,
|
|
"rewards/margins": 0.00189208984375,
|
|
"rewards/rejected": -0.0005645751953125,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.007816716606800543,
|
|
"grad_norm": 2.081361068965789,
|
|
"learning_rate": 3.9077037588388533e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -175.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.4312500059604645,
|
|
"rewards/chosen": 0.0027923583984375,
|
|
"rewards/margins": 0.0023651123046875,
|
|
"rewards/rejected": 0.000415802001953125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.008188941207124378,
|
|
"grad_norm": 1.7353430125248785,
|
|
"learning_rate": 4.093784890212133e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -160.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.00165557861328125,
|
|
"rewards/margins": 0.00142669677734375,
|
|
"rewards/rejected": 0.0002346038818359375,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.008561165807448215,
|
|
"grad_norm": 1.8793193185098336,
|
|
"learning_rate": 4.2798660215854113e-08,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": -0.0012054443359375,
|
|
"rewards/margins": -0.000732421875,
|
|
"rewards/rejected": -0.000469207763671875,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.00893339040777205,
|
|
"grad_norm": 1.5991670999722476,
|
|
"learning_rate": 4.46594715295869e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -156.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.38749998807907104,
|
|
"rewards/chosen": 0.00140380859375,
|
|
"rewards/margins": 0.00164031982421875,
|
|
"rewards/rejected": -0.00023555755615234375,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.009305615008095885,
|
|
"grad_norm": 1.8317490248709312,
|
|
"learning_rate": 4.652028284331969e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -182.0,
|
|
"logps/rejected": -177.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.41874998807907104,
|
|
"rewards/chosen": 0.00147247314453125,
|
|
"rewards/margins": 0.00092315673828125,
|
|
"rewards/rejected": 0.000545501708984375,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.00967783960841972,
|
|
"grad_norm": 1.8903059221093619,
|
|
"learning_rate": 4.8381094157052473e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -178.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.4437499940395355,
|
|
"rewards/chosen": 0.00148773193359375,
|
|
"rewards/margins": 0.002166748046875,
|
|
"rewards/rejected": -0.00067138671875,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.010050064208743556,
|
|
"grad_norm": 1.8628444868828153,
|
|
"learning_rate": 5.024190547078526e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -183.0,
|
|
"logps/rejected": -185.0,
|
|
"loss": 0.6918,
|
|
"rewards/accuracies": 0.4437499940395355,
|
|
"rewards/chosen": 0.001190185546875,
|
|
"rewards/margins": 0.0019073486328125,
|
|
"rewards/rejected": -0.000720977783203125,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.010422288809067391,
|
|
"grad_norm": 1.8657600217578134,
|
|
"learning_rate": 5.210271678451805e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -178.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": 0.00089263916015625,
|
|
"rewards/margins": 0.00157928466796875,
|
|
"rewards/rejected": -0.000690460205078125,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.010794513409391226,
|
|
"grad_norm": 1.8903717891223304,
|
|
"learning_rate": 5.3963528098250833e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -185.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.6918,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.002471923828125,
|
|
"rewards/margins": 0.0020294189453125,
|
|
"rewards/rejected": 0.0004405975341796875,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.011166738009715063,
|
|
"grad_norm": 1.8026622779801416,
|
|
"learning_rate": 5.582433941198362e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -185.0,
|
|
"logps/rejected": -174.0,
|
|
"loss": 0.6916,
|
|
"rewards/accuracies": 0.4437499940395355,
|
|
"rewards/chosen": 0.002899169921875,
|
|
"rewards/margins": 0.0024261474609375,
|
|
"rewards/rejected": 0.0004673004150390625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.011538962610038898,
|
|
"grad_norm": 1.8453511130821298,
|
|
"learning_rate": 5.7685150725716413e-08,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -186.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": 0.0035400390625,
|
|
"rewards/margins": 0.00054168701171875,
|
|
"rewards/rejected": 0.00299072265625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.011911187210362733,
|
|
"grad_norm": 1.7802917874049597,
|
|
"learning_rate": 5.95459620394492e-08,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -178.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6912,
|
|
"rewards/accuracies": 0.48750001192092896,
|
|
"rewards/chosen": 0.00457763671875,
|
|
"rewards/margins": 0.004241943359375,
|
|
"rewards/rejected": 0.00032806396484375,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.012283411810686568,
|
|
"grad_norm": 1.9319359790545254,
|
|
"learning_rate": 6.140677335318198e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.44999998807907104,
|
|
"rewards/chosen": 0.0047607421875,
|
|
"rewards/margins": 0.0016021728515625,
|
|
"rewards/rejected": 0.0031585693359375,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.012655636411010404,
|
|
"grad_norm": 1.8113164196825542,
|
|
"learning_rate": 6.326758466691477e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -164.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.48124998807907104,
|
|
"rewards/chosen": 0.0028839111328125,
|
|
"rewards/margins": 0.0020751953125,
|
|
"rewards/rejected": 0.000797271728515625,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.013027861011334239,
|
|
"grad_norm": 1.919143447401499,
|
|
"learning_rate": 6.512839598064757e-08,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -200.0,
|
|
"logps/rejected": -181.0,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.512499988079071,
|
|
"rewards/chosen": 0.00689697265625,
|
|
"rewards/margins": 0.0030364990234375,
|
|
"rewards/rejected": 0.003875732421875,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.013400085611658074,
|
|
"grad_norm": 1.9196252684766513,
|
|
"learning_rate": 6.698920729438035e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -182.0,
|
|
"logps/rejected": -167.0,
|
|
"loss": 0.6902,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": 0.0064697265625,
|
|
"rewards/margins": 0.0059814453125,
|
|
"rewards/rejected": 0.0004711151123046875,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.01377231021198191,
|
|
"grad_norm": 1.6718237175379975,
|
|
"learning_rate": 6.885001860811314e-08,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -174.0,
|
|
"logps/rejected": -171.0,
|
|
"loss": 0.6901,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.00848388671875,
|
|
"rewards/margins": 0.005584716796875,
|
|
"rewards/rejected": 0.00286865234375,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.014144534812305746,
|
|
"grad_norm": 1.8861986635424035,
|
|
"learning_rate": 7.071082992184592e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.69,
|
|
"rewards/accuracies": 0.4625000059604645,
|
|
"rewards/chosen": 0.0108642578125,
|
|
"rewards/margins": 0.002471923828125,
|
|
"rewards/rejected": 0.0084228515625,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.01451675941262958,
|
|
"grad_norm": 1.8157426593933548,
|
|
"learning_rate": 7.257164123557871e-08,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6899,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": 0.0093994140625,
|
|
"rewards/margins": 0.005035400390625,
|
|
"rewards/rejected": 0.004364013671875,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.014888984012953415,
|
|
"grad_norm": 1.9750645980727437,
|
|
"learning_rate": 7.44324525493115e-08,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -201.0,
|
|
"logps/rejected": -183.0,
|
|
"loss": 0.6893,
|
|
"rewards/accuracies": 0.5687500238418579,
|
|
"rewards/chosen": 0.011962890625,
|
|
"rewards/margins": 0.0078125,
|
|
"rewards/rejected": 0.004119873046875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.015261208613277252,
|
|
"grad_norm": 1.7364650095687586,
|
|
"learning_rate": 7.629326386304429e-08,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -193.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.689,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": 0.0111083984375,
|
|
"rewards/margins": 0.00665283203125,
|
|
"rewards/rejected": 0.004425048828125,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.015633433213601087,
|
|
"grad_norm": 1.8522890628366313,
|
|
"learning_rate": 7.815407517677707e-08,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -183.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6888,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": 0.01275634765625,
|
|
"rewards/margins": 0.0101318359375,
|
|
"rewards/rejected": 0.0025787353515625,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.016005657813924924,
|
|
"grad_norm": 1.6973792061969148,
|
|
"learning_rate": 8.001488649050986e-08,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -179.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6885,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 0.01416015625,
|
|
"rewards/margins": 0.00897216796875,
|
|
"rewards/rejected": 0.00518798828125,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.016377882414248757,
|
|
"grad_norm": 1.8728373803184584,
|
|
"learning_rate": 8.187569780424265e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -184.0,
|
|
"logps/rejected": -172.0,
|
|
"loss": 0.6885,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 0.0103759765625,
|
|
"rewards/margins": 0.0078125,
|
|
"rewards/rejected": 0.0025787353515625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.016750107014572593,
|
|
"grad_norm": 1.8412197892057298,
|
|
"learning_rate": 8.373650911797543e-08,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -179.0,
|
|
"logps/rejected": -164.0,
|
|
"loss": 0.6875,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.01129150390625,
|
|
"rewards/rejected": 0.004669189453125,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.01712233161489643,
|
|
"grad_norm": 1.9204236222643143,
|
|
"learning_rate": 8.559732043170823e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -193.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.688,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": 0.015869140625,
|
|
"rewards/margins": 0.008056640625,
|
|
"rewards/rejected": 0.00787353515625,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.017494556215220263,
|
|
"grad_norm": 1.894920797253737,
|
|
"learning_rate": 8.745813174544101e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -176.0,
|
|
"logps/rejected": -171.0,
|
|
"loss": 0.6881,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.012451171875,
|
|
"rewards/rejected": 0.003509521484375,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.0178667808155441,
|
|
"grad_norm": 1.7997148704363772,
|
|
"learning_rate": 8.93189430591738e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -185.0,
|
|
"loss": 0.6864,
|
|
"rewards/accuracies": 0.518750011920929,
|
|
"rewards/chosen": 0.01092529296875,
|
|
"rewards/margins": 0.0054931640625,
|
|
"rewards/rejected": 0.005462646484375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.018239005415867936,
|
|
"grad_norm": 1.6728829704360577,
|
|
"learning_rate": 9.117975437290658e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6872,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 0.01507568359375,
|
|
"rewards/margins": 0.01080322265625,
|
|
"rewards/rejected": 0.004241943359375,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.01861123001619177,
|
|
"grad_norm": 1.8678860471817902,
|
|
"learning_rate": 9.304056568663937e-08,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -202.0,
|
|
"logps/rejected": -182.0,
|
|
"loss": 0.6856,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 0.01202392578125,
|
|
"rewards/margins": 0.0157470703125,
|
|
"rewards/rejected": -0.0037689208984375,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.018983454616515606,
|
|
"grad_norm": 1.9112424142761917,
|
|
"learning_rate": 9.490137700037215e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -170.0,
|
|
"loss": 0.6854,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": 0.011474609375,
|
|
"rewards/margins": 0.01556396484375,
|
|
"rewards/rejected": -0.004058837890625,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.01935567921683944,
|
|
"grad_norm": 1.979614660486799,
|
|
"learning_rate": 9.676218831410495e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -200.0,
|
|
"logps/rejected": -181.0,
|
|
"loss": 0.6851,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.01251220703125,
|
|
"rewards/margins": 0.01416015625,
|
|
"rewards/rejected": -0.0015869140625,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.019727903817163276,
|
|
"grad_norm": 1.792134524724197,
|
|
"learning_rate": 9.862299962783774e-08,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -191.0,
|
|
"logps/rejected": -174.0,
|
|
"loss": 0.6856,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.01025390625,
|
|
"rewards/margins": 0.0133056640625,
|
|
"rewards/rejected": -0.0029754638671875,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.020100128417487113,
|
|
"grad_norm": 2.2953611386539396,
|
|
"learning_rate": 1.0048381094157052e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6841,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": 0.007049560546875,
|
|
"rewards/margins": 0.016845703125,
|
|
"rewards/rejected": -0.009765625,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.020472353017810946,
|
|
"grad_norm": 1.8662655435134705,
|
|
"learning_rate": 1.0234462225530331e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -191.0,
|
|
"logps/rejected": -186.0,
|
|
"loss": 0.6824,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.005523681640625,
|
|
"rewards/margins": 0.02099609375,
|
|
"rewards/rejected": -0.01544189453125,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.020844577618134782,
|
|
"grad_norm": 1.9136571833641907,
|
|
"learning_rate": 1.042054335690361e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -179.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.684,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.00830078125,
|
|
"rewards/margins": 0.014404296875,
|
|
"rewards/rejected": -0.006072998046875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.02121680221845862,
|
|
"grad_norm": 1.8371922091545678,
|
|
"learning_rate": 1.0606624488276889e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -177.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6811,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 0.0111083984375,
|
|
"rewards/margins": 0.0213623046875,
|
|
"rewards/rejected": -0.01031494140625,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.021589026818782452,
|
|
"grad_norm": 1.8850692074402484,
|
|
"learning_rate": 1.0792705619650167e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -180.0,
|
|
"loss": 0.6809,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.004852294921875,
|
|
"rewards/margins": 0.02490234375,
|
|
"rewards/rejected": -0.0201416015625,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.02196125141910629,
|
|
"grad_norm": 1.8515974929922816,
|
|
"learning_rate": 1.0978786751023446e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -181.0,
|
|
"loss": 0.6793,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": 0.0167236328125,
|
|
"rewards/margins": 0.0296630859375,
|
|
"rewards/rejected": -0.01300048828125,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.022333476019430126,
|
|
"grad_norm": 1.9579725733414266,
|
|
"learning_rate": 1.1164867882396724e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -179.0,
|
|
"loss": 0.6791,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": 0.006134033203125,
|
|
"rewards/margins": 0.0296630859375,
|
|
"rewards/rejected": -0.0235595703125,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.02270570061975396,
|
|
"grad_norm": 1.6937757035889593,
|
|
"learning_rate": 1.1350949013770003e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -149.0,
|
|
"loss": 0.679,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": 0.0037689208984375,
|
|
"rewards/margins": 0.03173828125,
|
|
"rewards/rejected": -0.0279541015625,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.023077925220077795,
|
|
"grad_norm": 1.9641278343114261,
|
|
"learning_rate": 1.1537030145143283e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -170.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6774,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.00555419921875,
|
|
"rewards/margins": 0.035400390625,
|
|
"rewards/rejected": -0.041015625,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.023450149820401632,
|
|
"grad_norm": 1.8818760286985359,
|
|
"learning_rate": 1.1723111276516561e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -176.0,
|
|
"logps/rejected": -159.0,
|
|
"loss": 0.6751,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.004852294921875,
|
|
"rewards/margins": 0.034912109375,
|
|
"rewards/rejected": -0.030029296875,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.023822374420725465,
|
|
"grad_norm": 2.0529907152598987,
|
|
"learning_rate": 1.190919240788984e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -178.0,
|
|
"loss": 0.6758,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.005157470703125,
|
|
"rewards/margins": 0.041748046875,
|
|
"rewards/rejected": -0.046875,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.024194599021049302,
|
|
"grad_norm": 1.986217709788575,
|
|
"learning_rate": 1.2095273539263117e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -192.0,
|
|
"logps/rejected": -180.0,
|
|
"loss": 0.6752,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.02001953125,
|
|
"rewards/margins": 0.037109375,
|
|
"rewards/rejected": -0.05712890625,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.024566823621373135,
|
|
"grad_norm": 1.9703999515170558,
|
|
"learning_rate": 1.2281354670636396e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -193.0,
|
|
"logps/rejected": -180.0,
|
|
"loss": 0.6727,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -0.024658203125,
|
|
"rewards/margins": 0.040283203125,
|
|
"rewards/rejected": -0.06494140625,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.02493904822169697,
|
|
"grad_norm": 2.264020590375023,
|
|
"learning_rate": 1.2467435802009675e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -192.0,
|
|
"logps/rejected": -182.0,
|
|
"loss": 0.6722,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.03515625,
|
|
"rewards/margins": 0.05126953125,
|
|
"rewards/rejected": -0.08642578125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.02531127282202081,
|
|
"grad_norm": 2.1399376124201086,
|
|
"learning_rate": 1.2653516933382955e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -215.0,
|
|
"logps/rejected": -187.0,
|
|
"loss": 0.669,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -0.042724609375,
|
|
"rewards/margins": 0.0294189453125,
|
|
"rewards/rejected": -0.07177734375,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.02568349742234464,
|
|
"grad_norm": 1.990696398679575,
|
|
"learning_rate": 1.2839598064756231e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -191.0,
|
|
"logps/rejected": -171.0,
|
|
"loss": 0.6685,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.050537109375,
|
|
"rewards/margins": 0.05029296875,
|
|
"rewards/rejected": -0.10107421875,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.026055722022668478,
|
|
"grad_norm": 1.9644294881089854,
|
|
"learning_rate": 1.3025679196129513e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -174.0,
|
|
"logps/rejected": -175.0,
|
|
"loss": 0.6672,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -0.06103515625,
|
|
"rewards/margins": 0.04638671875,
|
|
"rewards/rejected": -0.107421875,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.026427946622992315,
|
|
"grad_norm": 2.0626276391960907,
|
|
"learning_rate": 1.321176032750279e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -186.0,
|
|
"logps/rejected": -172.0,
|
|
"loss": 0.6664,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.06103515625,
|
|
"rewards/margins": 0.04736328125,
|
|
"rewards/rejected": -0.1083984375,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.026800171223316148,
|
|
"grad_norm": 2.0714043329819365,
|
|
"learning_rate": 1.339784145887607e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6649,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.064453125,
|
|
"rewards/margins": 0.076171875,
|
|
"rewards/rejected": -0.140625,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.027172395823639985,
|
|
"grad_norm": 2.0999461633523144,
|
|
"learning_rate": 1.3583922590249346e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -192.0,
|
|
"loss": 0.6609,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.078125,
|
|
"rewards/margins": 0.09033203125,
|
|
"rewards/rejected": -0.1689453125,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.02754462042396382,
|
|
"grad_norm": 1.9342673807765878,
|
|
"learning_rate": 1.3770003721622628e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6572,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -0.10888671875,
|
|
"rewards/margins": 0.05810546875,
|
|
"rewards/rejected": -0.1669921875,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.027916845024287654,
|
|
"grad_norm": 1.9998689351104397,
|
|
"learning_rate": 1.3956084852995905e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -227.0,
|
|
"logps/rejected": -191.0,
|
|
"loss": 0.6544,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.1162109375,
|
|
"rewards/margins": 0.0966796875,
|
|
"rewards/rejected": -0.212890625,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.02828906962461149,
|
|
"grad_norm": 2.2477309453236227,
|
|
"learning_rate": 1.4142165984369184e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -203.0,
|
|
"loss": 0.6513,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.134765625,
|
|
"rewards/margins": 0.10595703125,
|
|
"rewards/rejected": -0.2412109375,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.028661294224935328,
|
|
"grad_norm": 2.060954816992896,
|
|
"learning_rate": 1.432824711574246e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -179.0,
|
|
"loss": 0.6522,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -0.1669921875,
|
|
"rewards/margins": 0.08544921875,
|
|
"rewards/rejected": -0.251953125,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.02903351882525916,
|
|
"grad_norm": 2.1266109757052862,
|
|
"learning_rate": 1.4514328247115743e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -210.0,
|
|
"loss": 0.644,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.2060546875,
|
|
"rewards/margins": 0.119140625,
|
|
"rewards/rejected": -0.326171875,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.029405743425582997,
|
|
"grad_norm": 2.2396300152325614,
|
|
"learning_rate": 1.470040937848902e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -217.0,
|
|
"logps/rejected": -225.0,
|
|
"loss": 0.6371,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.228515625,
|
|
"rewards/margins": 0.1201171875,
|
|
"rewards/rejected": -0.34765625,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.02977796802590683,
|
|
"grad_norm": 2.2146992604089,
|
|
"learning_rate": 1.48864905098623e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -238.0,
|
|
"logps/rejected": -240.0,
|
|
"loss": 0.6448,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.2099609375,
|
|
"rewards/margins": 0.2001953125,
|
|
"rewards/rejected": -0.41015625,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.030150192626230667,
|
|
"grad_norm": 2.295329355568062,
|
|
"learning_rate": 1.5072571641235578e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -222.0,
|
|
"logps/rejected": -224.0,
|
|
"loss": 0.6383,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -0.302734375,
|
|
"rewards/margins": 0.142578125,
|
|
"rewards/rejected": -0.4453125,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.030522417226554504,
|
|
"grad_norm": 2.2993476230328076,
|
|
"learning_rate": 1.5258652772608857e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -202.0,
|
|
"logps/rejected": -199.0,
|
|
"loss": 0.637,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -0.35546875,
|
|
"rewards/margins": 0.0966796875,
|
|
"rewards/rejected": -0.453125,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.030894641826878337,
|
|
"grad_norm": 2.410686221160502,
|
|
"learning_rate": 1.5444733903982134e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -206.0,
|
|
"logps/rejected": -209.0,
|
|
"loss": 0.6276,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.359375,
|
|
"rewards/margins": 0.1572265625,
|
|
"rewards/rejected": -0.515625,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.031266866427202174,
|
|
"grad_norm": 2.351956208749687,
|
|
"learning_rate": 1.5630815035355413e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -218.0,
|
|
"logps/rejected": -216.0,
|
|
"loss": 0.6247,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.404296875,
|
|
"rewards/margins": 0.203125,
|
|
"rewards/rejected": -0.609375,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.03163909102752601,
|
|
"grad_norm": 2.204629059737825,
|
|
"learning_rate": 1.5816896166728693e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -197.0,
|
|
"logps/rejected": -207.0,
|
|
"loss": 0.6232,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.41796875,
|
|
"rewards/margins": 0.1904296875,
|
|
"rewards/rejected": -0.609375,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.03201131562784985,
|
|
"grad_norm": 2.4311568025578314,
|
|
"learning_rate": 1.6002977298101972e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -231.0,
|
|
"logps/rejected": -204.0,
|
|
"loss": 0.6281,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -0.423828125,
|
|
"rewards/margins": 0.123046875,
|
|
"rewards/rejected": -0.546875,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.03238354022817368,
|
|
"grad_norm": 2.5905040497602,
|
|
"learning_rate": 1.618905842947525e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -238.0,
|
|
"logps/rejected": -245.0,
|
|
"loss": 0.6243,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -0.4765625,
|
|
"rewards/margins": 0.19921875,
|
|
"rewards/rejected": -0.67578125,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.03275576482849751,
|
|
"grad_norm": 2.3496266988874495,
|
|
"learning_rate": 1.637513956084853e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -234.0,
|
|
"logps/rejected": -224.0,
|
|
"loss": 0.6269,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.50390625,
|
|
"rewards/margins": 0.158203125,
|
|
"rewards/rejected": -0.6640625,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.03312798942882135,
|
|
"grad_norm": 2.6617243292209003,
|
|
"learning_rate": 1.6561220692221807e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -221.0,
|
|
"logps/rejected": -223.0,
|
|
"loss": 0.6162,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.474609375,
|
|
"rewards/margins": 0.2001953125,
|
|
"rewards/rejected": -0.67578125,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.03350021402914519,
|
|
"grad_norm": 2.7726282183168927,
|
|
"learning_rate": 1.6747301823595087e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -233.0,
|
|
"logps/rejected": -238.0,
|
|
"loss": 0.6085,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.53515625,
|
|
"rewards/margins": 0.2431640625,
|
|
"rewards/rejected": -0.78125,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.03387243862946902,
|
|
"grad_norm": 2.706212443208337,
|
|
"learning_rate": 1.6933382954968363e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -244.0,
|
|
"logps/rejected": -262.0,
|
|
"loss": 0.5865,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.59765625,
|
|
"rewards/margins": 0.341796875,
|
|
"rewards/rejected": -0.9375,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.03424466322979286,
|
|
"grad_norm": 2.7082318302504023,
|
|
"learning_rate": 1.7119464086341645e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -241.0,
|
|
"logps/rejected": -249.0,
|
|
"loss": 0.5963,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.6484375,
|
|
"rewards/margins": 0.2490234375,
|
|
"rewards/rejected": -0.8984375,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.03461688783011669,
|
|
"grad_norm": 2.8004941194812867,
|
|
"learning_rate": 1.7305545217714922e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -252.0,
|
|
"logps/rejected": -248.0,
|
|
"loss": 0.5974,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -0.61328125,
|
|
"rewards/margins": 0.27734375,
|
|
"rewards/rejected": -0.890625,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.034989112430440526,
|
|
"grad_norm": 3.336638684956353,
|
|
"learning_rate": 1.7491626349088201e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -252.0,
|
|
"logps/rejected": -264.0,
|
|
"loss": 0.5936,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.69921875,
|
|
"rewards/margins": 0.265625,
|
|
"rewards/rejected": -0.9609375,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.03536133703076436,
|
|
"grad_norm": 3.2330647952855767,
|
|
"learning_rate": 1.767770748046148e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -255.0,
|
|
"logps/rejected": -276.0,
|
|
"loss": 0.5862,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.71875,
|
|
"rewards/margins": 0.306640625,
|
|
"rewards/rejected": -1.0234375,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.0357335616310882,
|
|
"grad_norm": 3.5863137960887053,
|
|
"learning_rate": 1.786378861183476e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -251.0,
|
|
"logps/rejected": -268.0,
|
|
"loss": 0.5854,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.75390625,
|
|
"rewards/margins": 0.30859375,
|
|
"rewards/rejected": -1.0625,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.036105786231412036,
|
|
"grad_norm": 3.1684550919691397,
|
|
"learning_rate": 1.8049869743208037e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -274.0,
|
|
"loss": 0.5866,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.7421875,
|
|
"rewards/margins": 0.259765625,
|
|
"rewards/rejected": -1.0,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.03647801083173587,
|
|
"grad_norm": 3.2428149852296877,
|
|
"learning_rate": 1.8235950874581316e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -278.0,
|
|
"loss": 0.5783,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.7109375,
|
|
"rewards/margins": 0.35546875,
|
|
"rewards/rejected": -1.0703125,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.0368502354320597,
|
|
"grad_norm": 3.164181259665228,
|
|
"learning_rate": 1.8422032005954595e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -300.0,
|
|
"loss": 0.5762,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.76171875,
|
|
"rewards/margins": 0.375,
|
|
"rewards/rejected": -1.1328125,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.03722246003238354,
|
|
"grad_norm": 3.4176535410789417,
|
|
"learning_rate": 1.8608113137327875e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -270.0,
|
|
"logps/rejected": -318.0,
|
|
"loss": 0.5697,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.82421875,
|
|
"rewards/margins": 0.3828125,
|
|
"rewards/rejected": -1.203125,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.037594684632707376,
|
|
"grad_norm": 4.727247211530869,
|
|
"learning_rate": 1.8794194268701151e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -270.0,
|
|
"logps/rejected": -302.0,
|
|
"loss": 0.5626,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.8359375,
|
|
"rewards/margins": 0.421875,
|
|
"rewards/rejected": -1.2578125,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.03796690923303121,
|
|
"grad_norm": 3.5772984638610925,
|
|
"learning_rate": 1.898027540007443e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -292.0,
|
|
"loss": 0.571,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -0.91015625,
|
|
"rewards/margins": 0.5,
|
|
"rewards/rejected": -1.4140625,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.03833913383335505,
|
|
"grad_norm": 4.044261559460498,
|
|
"learning_rate": 1.916635653144771e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -318.0,
|
|
"loss": 0.5623,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.92578125,
|
|
"rewards/margins": 0.5078125,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.03871135843367888,
|
|
"grad_norm": 3.442293008605092,
|
|
"learning_rate": 1.935243766282099e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -260.0,
|
|
"logps/rejected": -302.0,
|
|
"loss": 0.5731,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.890625,
|
|
"rewards/margins": 0.451171875,
|
|
"rewards/rejected": -1.34375,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.039083583034002715,
|
|
"grad_norm": 3.6629199991861467,
|
|
"learning_rate": 1.9538518794194266e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -328.0,
|
|
"loss": 0.5737,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.91015625,
|
|
"rewards/margins": 0.4453125,
|
|
"rewards/rejected": -1.359375,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.03945580763432655,
|
|
"grad_norm": 3.7278237031608388,
|
|
"learning_rate": 1.9724599925567548e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -266.0,
|
|
"logps/rejected": -304.0,
|
|
"loss": 0.563,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.890625,
|
|
"rewards/margins": 0.4296875,
|
|
"rewards/rejected": -1.3203125,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.03982803223465039,
|
|
"grad_norm": 4.909104297787964,
|
|
"learning_rate": 1.9910681056940825e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -346.0,
|
|
"loss": 0.5556,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.95703125,
|
|
"rewards/margins": 0.5703125,
|
|
"rewards/rejected": -1.53125,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.040200256834974225,
|
|
"grad_norm": 4.289178459607553,
|
|
"learning_rate": 2.0096762188314104e-07,
|
|
"logits/chosen": -3.296875,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -310.0,
|
|
"loss": 0.5649,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -1.015625,
|
|
"rewards/margins": 0.423828125,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.04057248143529806,
|
|
"grad_norm": 4.549142571625994,
|
|
"learning_rate": 2.028284331968738e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -320.0,
|
|
"loss": 0.5664,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.94921875,
|
|
"rewards/margins": 0.44921875,
|
|
"rewards/rejected": -1.3984375,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.04094470603562189,
|
|
"grad_norm": 4.304854151005621,
|
|
"learning_rate": 2.0468924451060663e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -314.0,
|
|
"loss": 0.5369,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.97265625,
|
|
"rewards/margins": 0.5859375,
|
|
"rewards/rejected": -1.5625,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.04131693063594573,
|
|
"grad_norm": 4.337380241949519,
|
|
"learning_rate": 2.065500558243394e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -324.0,
|
|
"loss": 0.5347,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.0234375,
|
|
"rewards/margins": 0.62109375,
|
|
"rewards/rejected": -1.640625,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.041689155236269565,
|
|
"grad_norm": 4.300189933067445,
|
|
"learning_rate": 2.084108671380722e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -308.0,
|
|
"loss": 0.5766,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.48046875,
|
|
"rewards/rejected": -1.578125,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.0420613798365934,
|
|
"grad_norm": 4.890466944643528,
|
|
"learning_rate": 2.1027167845180498e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -328.0,
|
|
"loss": 0.5407,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -0.8828125,
|
|
"rewards/margins": 0.66796875,
|
|
"rewards/rejected": -1.5546875,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.04243360443691724,
|
|
"grad_norm": 4.664611967902792,
|
|
"learning_rate": 2.1213248976553777e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.5594,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -1.0703125,
|
|
"rewards/margins": 0.435546875,
|
|
"rewards/rejected": -1.5078125,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.04280582903724107,
|
|
"grad_norm": 4.942468938353393,
|
|
"learning_rate": 2.1399330107927054e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.5503,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.1171875,
|
|
"rewards/margins": 0.482421875,
|
|
"rewards/rejected": -1.6015625,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.043178053637564905,
|
|
"grad_norm": 4.935776595126225,
|
|
"learning_rate": 2.1585411239300333e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.5471,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.078125,
|
|
"rewards/margins": 0.51953125,
|
|
"rewards/rejected": -1.6015625,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.04355027823788874,
|
|
"grad_norm": 4.793351753907498,
|
|
"learning_rate": 2.1771492370673613e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -320.0,
|
|
"loss": 0.554,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.9921875,
|
|
"rewards/margins": 0.53515625,
|
|
"rewards/rejected": -1.5234375,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.04392250283821258,
|
|
"grad_norm": 5.48400197202018,
|
|
"learning_rate": 2.1957573502046892e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -326.0,
|
|
"loss": 0.532,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.94921875,
|
|
"rewards/margins": 0.5625,
|
|
"rewards/rejected": -1.5078125,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.044294727438536415,
|
|
"grad_norm": 4.493529879184713,
|
|
"learning_rate": 2.214365463342017e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -326.0,
|
|
"loss": 0.5541,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.1015625,
|
|
"rewards/margins": 0.53515625,
|
|
"rewards/rejected": -1.640625,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.04466695203886025,
|
|
"grad_norm": 5.301914504399541,
|
|
"learning_rate": 2.2329735764793448e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -338.0,
|
|
"loss": 0.5421,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -1.0859375,
|
|
"rewards/margins": 0.57421875,
|
|
"rewards/rejected": -1.6640625,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.04503917663918408,
|
|
"grad_norm": 4.987000608823246,
|
|
"learning_rate": 2.2515816896166727e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -292.0,
|
|
"logps/rejected": -324.0,
|
|
"loss": 0.5489,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.451171875,
|
|
"rewards/rejected": -1.546875,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.04541140123950792,
|
|
"grad_norm": 5.708169792189986,
|
|
"learning_rate": 2.2701898027540007e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.5339,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -1.125,
|
|
"rewards/margins": 0.42578125,
|
|
"rewards/rejected": -1.546875,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.045783625839831754,
|
|
"grad_norm": 4.915909641193549,
|
|
"learning_rate": 2.2887979158913283e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -344.0,
|
|
"loss": 0.5376,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -1.1015625,
|
|
"rewards/margins": 0.6328125,
|
|
"rewards/rejected": -1.734375,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.04615585044015559,
|
|
"grad_norm": 4.255741576380326,
|
|
"learning_rate": 2.3074060290286565e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -344.0,
|
|
"loss": 0.5675,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -1.1484375,
|
|
"rewards/margins": 0.419921875,
|
|
"rewards/rejected": -1.5625,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.04652807504047943,
|
|
"grad_norm": 4.601416833791625,
|
|
"learning_rate": 2.3260141421659842e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.5461,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.5546875,
|
|
"rewards/rejected": -1.6484375,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.046900299640803264,
|
|
"grad_norm": 8.233795954417303,
|
|
"learning_rate": 2.3446222553033121e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -336.0,
|
|
"loss": 0.5647,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 0.45703125,
|
|
"rewards/rejected": -1.6171875,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.047272524241127094,
|
|
"grad_norm": 4.651834426567815,
|
|
"learning_rate": 2.3632303684406398e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -320.0,
|
|
"loss": 0.5355,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -1.140625,
|
|
"rewards/margins": 0.47265625,
|
|
"rewards/rejected": -1.609375,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.04764474884145093,
|
|
"grad_norm": 5.083390252185782,
|
|
"learning_rate": 2.381838481577968e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -332.0,
|
|
"loss": 0.5296,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.6015625,
|
|
"rewards/rejected": -1.6875,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.04801697344177477,
|
|
"grad_norm": 4.550219702551529,
|
|
"learning_rate": 2.4004465947152957e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -318.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.5355,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.5859375,
|
|
"rewards/rejected": -1.8203125,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.048389198042098604,
|
|
"grad_norm": 6.584872576687741,
|
|
"learning_rate": 2.4190547078526233e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.5441,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -1.2109375,
|
|
"rewards/margins": 0.734375,
|
|
"rewards/rejected": -1.9453125,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.04876142264242244,
|
|
"grad_norm": 5.354127669259726,
|
|
"learning_rate": 2.4376628209899515e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -322.0,
|
|
"loss": 0.5271,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.66015625,
|
|
"rewards/rejected": -1.7578125,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.04913364724274627,
|
|
"grad_norm": 6.293180352073142,
|
|
"learning_rate": 2.456270934127279e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -360.0,
|
|
"loss": 0.527,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.2265625,
|
|
"rewards/margins": 0.6328125,
|
|
"rewards/rejected": -1.859375,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.04950587184307011,
|
|
"grad_norm": 6.589478898199543,
|
|
"learning_rate": 2.4748790472646074e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -340.0,
|
|
"loss": 0.5578,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -1.3203125,
|
|
"rewards/margins": 0.40234375,
|
|
"rewards/rejected": -1.71875,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.04987809644339394,
|
|
"grad_norm": 5.835837740313499,
|
|
"learning_rate": 2.493487160401935e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -354.0,
|
|
"loss": 0.532,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -1.09375,
|
|
"rewards/margins": 0.609375,
|
|
"rewards/rejected": -1.6953125,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.05025032104371778,
|
|
"grad_norm": 5.826110770220833,
|
|
"learning_rate": 2.5120952735392633e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -360.0,
|
|
"loss": 0.5388,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -1.2578125,
|
|
"rewards/margins": 0.59375,
|
|
"rewards/rejected": -1.8515625,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.05062254564404162,
|
|
"grad_norm": 4.607455687504676,
|
|
"learning_rate": 2.530703386676591e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -336.0,
|
|
"logps/rejected": -366.0,
|
|
"loss": 0.546,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.3046875,
|
|
"rewards/margins": 0.515625,
|
|
"rewards/rejected": -1.8125,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.05099477024436545,
|
|
"grad_norm": 4.798281770709719,
|
|
"learning_rate": 2.5493114998139186e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.5213,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.2421875,
|
|
"rewards/margins": 0.6171875,
|
|
"rewards/rejected": -1.859375,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.05136699484468928,
|
|
"grad_norm": 5.911814051243962,
|
|
"learning_rate": 2.5679196129512463e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -318.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.5524,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.62109375,
|
|
"rewards/rejected": -1.859375,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.05173921944501312,
|
|
"grad_norm": 6.315439881014484,
|
|
"learning_rate": 2.5865277260885745e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.5407,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 0.73046875,
|
|
"rewards/rejected": -1.984375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.052111444045336956,
|
|
"grad_norm": 6.3816523341835625,
|
|
"learning_rate": 2.6051358392259027e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -384.0,
|
|
"loss": 0.5187,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 0.609375,
|
|
"rewards/rejected": -2.125,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.05248366864566079,
|
|
"grad_norm": 8.34074589234693,
|
|
"learning_rate": 2.6237439523632303e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -354.0,
|
|
"logps/rejected": -382.0,
|
|
"loss": 0.5349,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -1.546875,
|
|
"rewards/margins": 0.625,
|
|
"rewards/rejected": -2.171875,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.05285589324598463,
|
|
"grad_norm": 5.547059345045155,
|
|
"learning_rate": 2.642352065500558e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.5161,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -1.4375,
|
|
"rewards/margins": 0.71484375,
|
|
"rewards/rejected": -2.15625,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.05322811784630846,
|
|
"grad_norm": 6.194268883311141,
|
|
"learning_rate": 2.660960178637886e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -380.0,
|
|
"loss": 0.5082,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -1.46875,
|
|
"rewards/margins": 0.65234375,
|
|
"rewards/rejected": -2.125,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.053600342446632296,
|
|
"grad_norm": 6.28333908227948,
|
|
"learning_rate": 2.679568291775214e-07,
|
|
"logits/chosen": -3.46875,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.5071,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -1.6015625,
|
|
"rewards/margins": 0.8125,
|
|
"rewards/rejected": -2.421875,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.05397256704695613,
|
|
"grad_norm": 7.660789095769654,
|
|
"learning_rate": 2.6981764049125415e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.5023,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 0.7890625,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.05434479164727997,
|
|
"grad_norm": 6.3843861768941625,
|
|
"learning_rate": 2.716784518049869e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -400.0,
|
|
"loss": 0.5385,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 0.60546875,
|
|
"rewards/rejected": -2.3125,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.054717016247603806,
|
|
"grad_norm": 5.273430639524879,
|
|
"learning_rate": 2.735392631187198e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -380.0,
|
|
"loss": 0.5101,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -1.4296875,
|
|
"rewards/margins": 0.70703125,
|
|
"rewards/rejected": -2.140625,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.05508924084792764,
|
|
"grad_norm": 6.68285628738834,
|
|
"learning_rate": 2.7540007443245256e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -370.0,
|
|
"logps/rejected": -434.0,
|
|
"loss": 0.504,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 0.76953125,
|
|
"rewards/rejected": -2.53125,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.05546146544825147,
|
|
"grad_norm": 5.618700604924483,
|
|
"learning_rate": 2.7726088574618533e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.512,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.6171875,
|
|
"rewards/margins": 0.77734375,
|
|
"rewards/rejected": -2.40625,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.05583369004857531,
|
|
"grad_norm": 7.385793453071723,
|
|
"learning_rate": 2.791216970599181e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -380.0,
|
|
"loss": 0.5139,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.4140625,
|
|
"rewards/margins": 0.84375,
|
|
"rewards/rejected": -2.25,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.056205914648899145,
|
|
"grad_norm": 9.131881751521462,
|
|
"learning_rate": 2.809825083736509e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -372.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.5095,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 0.9140625,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.05657813924922298,
|
|
"grad_norm": 5.557968050400609,
|
|
"learning_rate": 2.828433196873837e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -412.0,
|
|
"loss": 0.5186,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -1.6484375,
|
|
"rewards/margins": 0.80859375,
|
|
"rewards/rejected": -2.453125,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.05695036384954682,
|
|
"grad_norm": 8.992491744756656,
|
|
"learning_rate": 2.8470413100111645e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.4997,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -1.5078125,
|
|
"rewards/margins": 0.765625,
|
|
"rewards/rejected": -2.28125,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.057322588449870655,
|
|
"grad_norm": 7.731748608613605,
|
|
"learning_rate": 2.865649423148492e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -378.0,
|
|
"logps/rejected": -448.0,
|
|
"loss": 0.5122,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -1.9296875,
|
|
"rewards/margins": 0.7109375,
|
|
"rewards/rejected": -2.640625,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.057694813050194485,
|
|
"grad_norm": 5.311753845164432,
|
|
"learning_rate": 2.884257536285821e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -400.0,
|
|
"loss": 0.506,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 0.9140625,
|
|
"rewards/rejected": -2.4375,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.05806703765051832,
|
|
"grad_norm": 5.32907569317797,
|
|
"learning_rate": 2.9028656494231485e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.5211,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 0.8359375,
|
|
"rewards/rejected": -2.484375,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.05843926225084216,
|
|
"grad_norm": 7.635655915842599,
|
|
"learning_rate": 2.921473762560476e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -414.0,
|
|
"loss": 0.5095,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.6875,
|
|
"rewards/margins": 0.8828125,
|
|
"rewards/rejected": -2.5625,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.058811486851165995,
|
|
"grad_norm": 6.608327844564691,
|
|
"learning_rate": 2.940081875697804e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -410.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.5039,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -2.015625,
|
|
"rewards/margins": 0.7578125,
|
|
"rewards/rejected": -2.765625,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.05918371145148983,
|
|
"grad_norm": 6.778090332548805,
|
|
"learning_rate": 2.958689988835132e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -376.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4881,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -1.921875,
|
|
"rewards/margins": 0.82421875,
|
|
"rewards/rejected": -2.75,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.05955593605181366,
|
|
"grad_norm": 5.971408337951184,
|
|
"learning_rate": 2.97729810197246e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -418.0,
|
|
"loss": 0.503,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -1.7890625,
|
|
"rewards/margins": 0.88671875,
|
|
"rewards/rejected": -2.671875,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.0599281606521375,
|
|
"grad_norm": 6.789859874378571,
|
|
"learning_rate": 2.9959062151097874e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -440.0,
|
|
"loss": 0.4997,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.8359375,
|
|
"rewards/margins": 0.85546875,
|
|
"rewards/rejected": -2.6875,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.060300385252461335,
|
|
"grad_norm": 6.166048650905424,
|
|
"learning_rate": 3.0145143282471156e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -352.0,
|
|
"logps/rejected": -428.0,
|
|
"loss": 0.5049,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 0.78125,
|
|
"rewards/rejected": -2.421875,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.06067260985278517,
|
|
"grad_norm": 6.101789918238695,
|
|
"learning_rate": 3.033122441384444e-07,
|
|
"logits/chosen": -3.359375,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -370.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.4974,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -2.03125,
|
|
"rewards/margins": 0.88671875,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.06104483445310901,
|
|
"grad_norm": 6.135484413003626,
|
|
"learning_rate": 3.0517305545217715e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -368.0,
|
|
"logps/rejected": -430.0,
|
|
"loss": 0.493,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -1.859375,
|
|
"rewards/margins": 0.859375,
|
|
"rewards/rejected": -2.71875,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.061417059053432844,
|
|
"grad_norm": 5.505631441835803,
|
|
"learning_rate": 3.070338667659099e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -392.0,
|
|
"logps/rejected": -448.0,
|
|
"loss": 0.5239,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -2.046875,
|
|
"rewards/margins": 0.75390625,
|
|
"rewards/rejected": -2.796875,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.061789283653756674,
|
|
"grad_norm": 12.098147126661164,
|
|
"learning_rate": 3.088946780796427e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.4813,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -2.09375,
|
|
"rewards/margins": 0.8828125,
|
|
"rewards/rejected": -2.984375,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.06216150825408051,
|
|
"grad_norm": 6.772250695058595,
|
|
"learning_rate": 3.107554893933755e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -446.0,
|
|
"loss": 0.5249,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -1.8984375,
|
|
"rewards/margins": 0.88671875,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.06253373285440435,
|
|
"grad_norm": 5.468142537091393,
|
|
"learning_rate": 3.1261630070710827e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -432.0,
|
|
"loss": 0.5078,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -1.9375,
|
|
"rewards/margins": 0.69921875,
|
|
"rewards/rejected": -2.640625,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.06290595745472818,
|
|
"grad_norm": 7.741085979284499,
|
|
"learning_rate": 3.144771120208411e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.4899,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 0.98828125,
|
|
"rewards/rejected": -2.859375,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.06327818205505202,
|
|
"grad_norm": 5.958263874369148,
|
|
"learning_rate": 3.1633792333457385e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4978,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -2.03125,
|
|
"rewards/margins": 0.9453125,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.06365040665537586,
|
|
"grad_norm": 6.501126511734826,
|
|
"learning_rate": 3.181987346483067e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -396.0,
|
|
"logps/rejected": -458.0,
|
|
"loss": 0.5076,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -2.109375,
|
|
"rewards/margins": 0.7890625,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.0640226312556997,
|
|
"grad_norm": 6.27027575089657,
|
|
"learning_rate": 3.2005954596203944e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -400.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4805,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -2.265625,
|
|
"rewards/margins": 0.83984375,
|
|
"rewards/rejected": -3.09375,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.06439485585602353,
|
|
"grad_norm": 6.106140440630774,
|
|
"learning_rate": 3.219203572757722e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -378.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.5104,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -2.0625,
|
|
"rewards/margins": 0.84765625,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.06476708045634735,
|
|
"grad_norm": 5.9280110995386215,
|
|
"learning_rate": 3.23781168589505e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -368.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.4954,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.84375,
|
|
"rewards/margins": 0.97265625,
|
|
"rewards/rejected": -2.8125,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.06513930505667119,
|
|
"grad_norm": 6.574248401473447,
|
|
"learning_rate": 3.256419799032378e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -366.0,
|
|
"logps/rejected": -414.0,
|
|
"loss": 0.5184,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 0.66796875,
|
|
"rewards/rejected": -2.421875,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.06551152965699503,
|
|
"grad_norm": 5.849233018979067,
|
|
"learning_rate": 3.275027912169706e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.4899,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -2.046875,
|
|
"rewards/margins": 0.875,
|
|
"rewards/rejected": -2.921875,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.06588375425731886,
|
|
"grad_norm": 5.8974565519068305,
|
|
"learning_rate": 3.293636025307034e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -508.0,
|
|
"loss": 0.4984,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -2.453125,
|
|
"rewards/margins": 0.921875,
|
|
"rewards/rejected": -3.375,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.0662559788576427,
|
|
"grad_norm": 8.018391725298722,
|
|
"learning_rate": 3.3122441384443615e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -370.0,
|
|
"logps/rejected": -430.0,
|
|
"loss": 0.5038,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.828125,
|
|
"rewards/margins": 0.85546875,
|
|
"rewards/rejected": -2.6875,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.06662820345796654,
|
|
"grad_norm": 6.219744150368751,
|
|
"learning_rate": 3.3308522515816897e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -384.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.4941,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -2.015625,
|
|
"rewards/margins": 0.85546875,
|
|
"rewards/rejected": -2.875,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.06700042805829037,
|
|
"grad_norm": 10.757972805570088,
|
|
"learning_rate": 3.3494603647190173e-07,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -524.0,
|
|
"loss": 0.5043,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -2.65625,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -3.546875,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.06737265265861421,
|
|
"grad_norm": 7.793214262783194,
|
|
"learning_rate": 3.368068477856345e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -484.0,
|
|
"loss": 0.4779,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -2.4375,
|
|
"rewards/margins": 0.828125,
|
|
"rewards/rejected": -3.265625,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.06774487725893805,
|
|
"grad_norm": 7.52057370608622,
|
|
"learning_rate": 3.3866765909936727e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -392.0,
|
|
"logps/rejected": -460.0,
|
|
"loss": 0.4898,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.90625,
|
|
"rewards/margins": 1.015625,
|
|
"rewards/rejected": -2.921875,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.06811710185926188,
|
|
"grad_norm": 6.76354734193844,
|
|
"learning_rate": 3.4052847041310014e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -380.0,
|
|
"logps/rejected": -458.0,
|
|
"loss": 0.4937,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -2.125,
|
|
"rewards/margins": 0.91015625,
|
|
"rewards/rejected": -3.03125,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.06848932645958572,
|
|
"grad_norm": 7.56921741102404,
|
|
"learning_rate": 3.423892817268329e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.4962,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.140625,
|
|
"rewards/margins": 0.97265625,
|
|
"rewards/rejected": -3.109375,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.06886155105990954,
|
|
"grad_norm": 9.080547138345112,
|
|
"learning_rate": 3.442500930405657e-07,
|
|
"logits/chosen": -3.59375,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.4815,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -2.25,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -3.15625,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.06923377566023338,
|
|
"grad_norm": 6.29737254648851,
|
|
"learning_rate": 3.4611090435429844e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -490.0,
|
|
"loss": 0.4914,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 0.88671875,
|
|
"rewards/rejected": -3.109375,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.06960600026055722,
|
|
"grad_norm": 7.517613218540249,
|
|
"learning_rate": 3.4797171566803126e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4652,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -2.15625,
|
|
"rewards/margins": 0.859375,
|
|
"rewards/rejected": -3.0,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.06997822486088105,
|
|
"grad_norm": 11.144996982874401,
|
|
"learning_rate": 3.4983252698176403e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.5077,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -2.53125,
|
|
"rewards/margins": 0.8828125,
|
|
"rewards/rejected": -3.40625,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.07035044946120489,
|
|
"grad_norm": 7.801867883753414,
|
|
"learning_rate": 3.516933382954968e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -400.0,
|
|
"logps/rejected": -478.0,
|
|
"loss": 0.4787,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.09375,
|
|
"rewards/margins": 0.91796875,
|
|
"rewards/rejected": -3.015625,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.07072267406152873,
|
|
"grad_norm": 7.7839330396146895,
|
|
"learning_rate": 3.535541496092296e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.4892,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 0.89453125,
|
|
"rewards/rejected": -3.578125,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.07109489866185256,
|
|
"grad_norm": 6.578620581724611,
|
|
"learning_rate": 3.5541496092296243e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -524.0,
|
|
"loss": 0.4962,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.484375,
|
|
"rewards/margins": 0.96875,
|
|
"rewards/rejected": -3.453125,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.0714671232621764,
|
|
"grad_norm": 6.272864465633893,
|
|
"learning_rate": 3.572757722366952e-07,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -388.0,
|
|
"logps/rejected": -460.0,
|
|
"loss": 0.4991,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 0.7734375,
|
|
"rewards/rejected": -3.015625,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.07183934786250024,
|
|
"grad_norm": 8.192434922872225,
|
|
"learning_rate": 3.5913658355042797e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -496.0,
|
|
"loss": 0.503,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 0.734375,
|
|
"rewards/rejected": -3.234375,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.07221157246282407,
|
|
"grad_norm": 8.266227005876804,
|
|
"learning_rate": 3.6099739486416073e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -540.0,
|
|
"loss": 0.5006,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 0.8515625,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.07258379706314791,
|
|
"grad_norm": 5.648100260022086,
|
|
"learning_rate": 3.6285820617789355e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.5073,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -2.3125,
|
|
"rewards/margins": 0.8203125,
|
|
"rewards/rejected": -3.125,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.07295602166347175,
|
|
"grad_norm": 8.402952917263368,
|
|
"learning_rate": 3.647190174916263e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -504.0,
|
|
"loss": 0.5016,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 0.7734375,
|
|
"rewards/rejected": -3.09375,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.07332824626379557,
|
|
"grad_norm": 8.519398872873577,
|
|
"learning_rate": 3.665798288053591e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -408.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4778,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -2.375,
|
|
"rewards/margins": 0.921875,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.0737004708641194,
|
|
"grad_norm": 6.0929979281222915,
|
|
"learning_rate": 3.684406401190919e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -406.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.5153,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -2.21875,
|
|
"rewards/margins": 0.84765625,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.07407269546444324,
|
|
"grad_norm": 7.579804731883034,
|
|
"learning_rate": 3.7030145143282473e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.4795,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -2.5625,
|
|
"rewards/margins": 0.9609375,
|
|
"rewards/rejected": -3.53125,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.07444492006476708,
|
|
"grad_norm": 6.777010716505804,
|
|
"learning_rate": 3.721622627465575e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -552.0,
|
|
"loss": 0.4725,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 1.0,
|
|
"rewards/rejected": -3.75,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.07481714466509091,
|
|
"grad_norm": 7.877962084278623,
|
|
"learning_rate": 3.7402307406029026e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.5049,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.34375,
|
|
"rewards/margins": 0.8828125,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.07518936926541475,
|
|
"grad_norm": 6.5521323559610245,
|
|
"learning_rate": 3.7588388537402303e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.4811,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -2.265625,
|
|
"rewards/margins": 0.95703125,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.07556159386573859,
|
|
"grad_norm": 10.371968651489745,
|
|
"learning_rate": 3.7774469668775585e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -498.0,
|
|
"loss": 0.4834,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -2.734375,
|
|
"rewards/margins": 0.9140625,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.07593381846606242,
|
|
"grad_norm": 7.268850054113197,
|
|
"learning_rate": 3.796055080014886e-07,
|
|
"logits/chosen": -3.46875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.4965,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 0.8359375,
|
|
"rewards/rejected": -3.515625,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.07630604306638626,
|
|
"grad_norm": 9.101765248024416,
|
|
"learning_rate": 3.8146631931522143e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -548.0,
|
|
"loss": 0.4665,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -2.671875,
|
|
"rewards/margins": 1.15625,
|
|
"rewards/rejected": -3.8125,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.0766782676667101,
|
|
"grad_norm": 7.977249857002326,
|
|
"learning_rate": 3.833271306289542e-07,
|
|
"logits/chosen": -3.484375,
|
|
"logits/rejected": -3.3125,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -544.0,
|
|
"loss": 0.4635,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -2.890625,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -4.0,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.07705049226703393,
|
|
"grad_norm": 8.285848628565708,
|
|
"learning_rate": 3.85187941942687e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -552.0,
|
|
"loss": 0.4919,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -2.59375,
|
|
"rewards/margins": 1.0,
|
|
"rewards/rejected": -3.59375,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.07742271686735776,
|
|
"grad_norm": 7.660587395619919,
|
|
"learning_rate": 3.870487532564198e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -544.0,
|
|
"loss": 0.4886,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 0.89453125,
|
|
"rewards/rejected": -3.734375,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.0777949414676816,
|
|
"grad_norm": 6.2914110312102,
|
|
"learning_rate": 3.8890956457015255e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -516.0,
|
|
"loss": 0.4788,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 0.7890625,
|
|
"rewards/rejected": -3.421875,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.07816716606800543,
|
|
"grad_norm": 6.612583563249112,
|
|
"learning_rate": 3.907703758838853e-07,
|
|
"logits/chosen": -3.5,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -568.0,
|
|
"loss": 0.4708,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.875,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -3.984375,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.07853939066832927,
|
|
"grad_norm": 7.541886875709588,
|
|
"learning_rate": 3.9263118719761814e-07,
|
|
"logits/chosen": -3.53125,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -588.0,
|
|
"loss": 0.4718,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.0789116152686531,
|
|
"grad_norm": 7.403493751575397,
|
|
"learning_rate": 3.9449199851135096e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -486.0,
|
|
"logps/rejected": -556.0,
|
|
"loss": 0.4998,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -4.03125,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.07928383986897694,
|
|
"grad_norm": 7.636053360461304,
|
|
"learning_rate": 3.9635280982508373e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.4606,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -2.671875,
|
|
"rewards/margins": 0.92578125,
|
|
"rewards/rejected": -3.59375,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.07965606446930078,
|
|
"grad_norm": 6.995222207530676,
|
|
"learning_rate": 3.982136211388165e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -576.0,
|
|
"loss": 0.4656,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -3.015625,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.08002828906962461,
|
|
"grad_norm": 6.35027390290969,
|
|
"learning_rate": 4.000744324525493e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.4815,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.890625,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -3.96875,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.08040051366994845,
|
|
"grad_norm": 8.305965505985583,
|
|
"learning_rate": 4.019352437662821e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -528.0,
|
|
"loss": 0.4668,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 0.85546875,
|
|
"rewards/rejected": -3.5625,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.08077273827027229,
|
|
"grad_norm": 9.07576399707983,
|
|
"learning_rate": 4.0379605508001485e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -600.0,
|
|
"loss": 0.4852,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -4.25,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.08114496287059612,
|
|
"grad_norm": 6.12625628924914,
|
|
"learning_rate": 4.056568663937476e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.4739,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.08151718747091995,
|
|
"grad_norm": 6.544507966611785,
|
|
"learning_rate": 4.075176777074805e-07,
|
|
"logits/chosen": -3.453125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -532.0,
|
|
"loss": 0.4782,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 0.99609375,
|
|
"rewards/rejected": -3.59375,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.08188941207124378,
|
|
"grad_norm": 5.385034258739545,
|
|
"learning_rate": 4.0937848902121325e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.4817,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 0.859375,
|
|
"rewards/rejected": -4.3125,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.08226163667156762,
|
|
"grad_norm": 6.5116283708203735,
|
|
"learning_rate": 4.11239300334946e-07,
|
|
"logits/chosen": -3.59375,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -540.0,
|
|
"loss": 0.4893,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 0.75,
|
|
"rewards/rejected": -3.859375,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.08263386127189146,
|
|
"grad_norm": 8.889763195218695,
|
|
"learning_rate": 4.131001116486788e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -576.0,
|
|
"loss": 0.4677,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -3.171875,
|
|
"rewards/margins": 0.890625,
|
|
"rewards/rejected": -4.0625,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.0830060858722153,
|
|
"grad_norm": 6.614280229899724,
|
|
"learning_rate": 4.149609229624116e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.4802,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.08337831047253913,
|
|
"grad_norm": 6.203608504808656,
|
|
"learning_rate": 4.168217342761444e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -548.0,
|
|
"loss": 0.4752,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -2.890625,
|
|
"rewards/margins": 1.0390625,
|
|
"rewards/rejected": -3.921875,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.08375053507286297,
|
|
"grad_norm": 8.792204618142273,
|
|
"learning_rate": 4.1868254558987714e-07,
|
|
"logits/chosen": -3.515625,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -548.0,
|
|
"loss": 0.4787,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 0.84375,
|
|
"rewards/rejected": -3.90625,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.0841227596731868,
|
|
"grad_norm": 8.883928868598451,
|
|
"learning_rate": 4.2054335690360996e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -580.0,
|
|
"loss": 0.4764,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 1.03125,
|
|
"rewards/rejected": -4.0,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.08449498427351064,
|
|
"grad_norm": 8.003135021321002,
|
|
"learning_rate": 4.224041682173428e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -508.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.4988,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 0.84765625,
|
|
"rewards/rejected": -4.125,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.08486720887383448,
|
|
"grad_norm": 5.65226505535633,
|
|
"learning_rate": 4.2426497953107555e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.468,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 0.9296875,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.08523943347415831,
|
|
"grad_norm": 6.179490889540778,
|
|
"learning_rate": 4.261257908448083e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.4939,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 1.046875,
|
|
"rewards/rejected": -4.25,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.08561165807448214,
|
|
"grad_norm": 5.801450335577568,
|
|
"learning_rate": 4.279866021585411e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -588.0,
|
|
"loss": 0.4774,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 0.953125,
|
|
"rewards/rejected": -4.15625,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.08598388267480597,
|
|
"grad_norm": 6.04888665530274,
|
|
"learning_rate": 4.298474134722739e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.4416,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -3.578125,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.08635610727512981,
|
|
"grad_norm": 7.9370533049546435,
|
|
"learning_rate": 4.3170822478600667e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.473,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.08672833187545365,
|
|
"grad_norm": 7.029632975823499,
|
|
"learning_rate": 4.335690360997395e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.4548,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.08710055647577748,
|
|
"grad_norm": 8.542576339215444,
|
|
"learning_rate": 4.3542984741347225e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.4731,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 1.0859375,
|
|
"rewards/rejected": -4.8125,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.08747278107610132,
|
|
"grad_norm": 6.607106523171658,
|
|
"learning_rate": 4.372906587272051e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.4753,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.08784500567642516,
|
|
"grad_norm": 6.991018012972489,
|
|
"learning_rate": 4.3915147004093784e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -600.0,
|
|
"loss": 0.4553,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.703125,
|
|
"rewards/margins": 0.8359375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.08821723027674899,
|
|
"grad_norm": 7.676829082785754,
|
|
"learning_rate": 4.410122813546706e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.4849,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 1.0,
|
|
"rewards/rejected": -4.15625,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.08858945487707283,
|
|
"grad_norm": 7.527652000753536,
|
|
"learning_rate": 4.428730926684034e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -588.0,
|
|
"loss": 0.5043,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 0.89453125,
|
|
"rewards/rejected": -4.1875,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.08896167947739667,
|
|
"grad_norm": 5.531822712679618,
|
|
"learning_rate": 4.447339039821362e-07,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.457,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -3.59375,
|
|
"rewards/margins": 1.15625,
|
|
"rewards/rejected": -4.75,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.0893339040777205,
|
|
"grad_norm": 6.682482626969178,
|
|
"learning_rate": 4.4659471529586896e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -600.0,
|
|
"loss": 0.4789,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.08970612867804433,
|
|
"grad_norm": 8.302427926878842,
|
|
"learning_rate": 4.484555266096018e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.4751,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -4.625,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.09007835327836816,
|
|
"grad_norm": 5.977955107350341,
|
|
"learning_rate": 4.5031633792333455e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.4853,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 1.0703125,
|
|
"rewards/rejected": -4.78125,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.090450577878692,
|
|
"grad_norm": 6.23030548847072,
|
|
"learning_rate": 4.5217714923706737e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -600.0,
|
|
"loss": 0.4668,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.171875,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.09082280247901584,
|
|
"grad_norm": 7.091500663393886,
|
|
"learning_rate": 4.5403796055080013e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.4546,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -4.5,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.09119502707933967,
|
|
"grad_norm": 7.004741424516515,
|
|
"learning_rate": 4.558987718645329e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.4576,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 1.046875,
|
|
"rewards/rejected": -4.375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.09156725167966351,
|
|
"grad_norm": 6.611813344962854,
|
|
"learning_rate": 4.5775958317826567e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.4805,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.53125,
|
|
"rewards/margins": 0.91796875,
|
|
"rewards/rejected": -4.4375,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.09193947627998734,
|
|
"grad_norm": 6.416497922144062,
|
|
"learning_rate": 4.596203944919985e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.4841,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.09231170088031118,
|
|
"grad_norm": 6.102285637173607,
|
|
"learning_rate": 4.614812058057313e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -608.0,
|
|
"loss": 0.4645,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 0.89453125,
|
|
"rewards/rejected": -4.4375,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.09268392548063502,
|
|
"grad_norm": 5.718071657531197,
|
|
"learning_rate": 4.633420171194641e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -620.0,
|
|
"loss": 0.4933,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 0.92578125,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.09305615008095885,
|
|
"grad_norm": 6.298059347419434,
|
|
"learning_rate": 4.6520282843319684e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.4895,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.59375,
|
|
"rewards/margins": 0.9609375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.09342837468128269,
|
|
"grad_norm": 7.644118403626093,
|
|
"learning_rate": 4.6706363974692966e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.5008,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -4.8125,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.09380059928160653,
|
|
"grad_norm": 7.371156916827325,
|
|
"learning_rate": 4.6892445106066243e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.471,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 0.921875,
|
|
"rewards/rejected": -4.375,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.09417282388193035,
|
|
"grad_norm": 5.053817248660392,
|
|
"learning_rate": 4.707852623743952e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.4944,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -5.0,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.09454504848225419,
|
|
"grad_norm": 6.609709896985202,
|
|
"learning_rate": 4.7264607368812796e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.4804,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 1.015625,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.09491727308257802,
|
|
"grad_norm": 7.593526504461849,
|
|
"learning_rate": 4.7450688500186083e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.4454,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 0.9765625,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.09528949768290186,
|
|
"grad_norm": 8.46313028709872,
|
|
"learning_rate": 4.763676963155936e-07,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.4735,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 1.15625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.0956617222832257,
|
|
"grad_norm": 8.179385105457904,
|
|
"learning_rate": 4.782285076293264e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.4653,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -4.4375,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.09603394688354953,
|
|
"grad_norm": 5.882894822920484,
|
|
"learning_rate": 4.800893189430591e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.4578,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -3.78125,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -4.875,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.09640617148387337,
|
|
"grad_norm": 6.136891241565959,
|
|
"learning_rate": 4.81950130256792e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.4623,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.09677839608419721,
|
|
"grad_norm": 5.995934797811196,
|
|
"learning_rate": 4.838109415705247e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -632.0,
|
|
"loss": 0.4682,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 0.92578125,
|
|
"rewards/rejected": -4.65625,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.09715062068452104,
|
|
"grad_norm": 8.39140037485633,
|
|
"learning_rate": 4.856717528842575e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -640.0,
|
|
"loss": 0.4679,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 0.9375,
|
|
"rewards/rejected": -4.78125,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.09752284528484488,
|
|
"grad_norm": 6.728658542049892,
|
|
"learning_rate": 4.875325641979903e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.4658,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.09789506988516872,
|
|
"grad_norm": 5.961291210522545,
|
|
"learning_rate": 4.893933755117231e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.4693,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -4.3125,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.09826729448549254,
|
|
"grad_norm": 6.374922100398644,
|
|
"learning_rate": 4.912541868254558e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.4851,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.09863951908581638,
|
|
"grad_norm": 6.008215668867427,
|
|
"learning_rate": 4.931149981391887e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.4813,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 0.91015625,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.09901174368614021,
|
|
"grad_norm": 7.095347167166638,
|
|
"learning_rate": 4.949758094529215e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -652.0,
|
|
"loss": 0.4847,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -5.0,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.09938396828646405,
|
|
"grad_norm": 6.5667446269543115,
|
|
"learning_rate": 4.968366207666542e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.4679,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -3.890625,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.09975619288678789,
|
|
"grad_norm": 7.083716586923517,
|
|
"learning_rate": 4.98697432080387e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.4665,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.10012841748711172,
|
|
"grad_norm": 6.254478167134791,
|
|
"learning_rate": 4.999999810062151e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.4631,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 0.84375,
|
|
"rewards/rejected": -5.0,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.10050064208743556,
|
|
"grad_norm": 7.351190394023517,
|
|
"learning_rate": 4.99999643339008e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.4632,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -5.25,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.1008728666877594,
|
|
"grad_norm": 7.912122090881811,
|
|
"learning_rate": 4.999988835883477e-07,
|
|
"logits/chosen": -3.59375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.4627,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.10124509128808323,
|
|
"grad_norm": 9.566048432876075,
|
|
"learning_rate": 4.999977017555171e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.4796,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 0.8515625,
|
|
"rewards/rejected": -4.0,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.10161731588840707,
|
|
"grad_norm": 6.662788071697429,
|
|
"learning_rate": 4.999960978425113e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -620.0,
|
|
"loss": 0.4441,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -4.5,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.1019895404887309,
|
|
"grad_norm": 7.872813273518058,
|
|
"learning_rate": 4.999940718520385e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.4778,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 1.1328125,
|
|
"rewards/rejected": -5.03125,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.10236176508905473,
|
|
"grad_norm": 5.599470744457507,
|
|
"learning_rate": 4.999916237875191e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.4441,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.10273398968937857,
|
|
"grad_norm": 6.068554789034378,
|
|
"learning_rate": 4.999887536530864e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.4644,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.1031062142897024,
|
|
"grad_norm": 6.028131826751676,
|
|
"learning_rate": 4.999854614535859e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.4599,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 0.91015625,
|
|
"rewards/rejected": -5.1875,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.10347843889002624,
|
|
"grad_norm": 6.558263540906065,
|
|
"learning_rate": 4.999817471945762e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.4716,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 0.9140625,
|
|
"rewards/rejected": -4.875,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.10385066349035008,
|
|
"grad_norm": 6.6512482573387635,
|
|
"learning_rate": 4.99977610882328e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.4511,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -5.1875,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.10422288809067391,
|
|
"grad_norm": 6.478428424893469,
|
|
"learning_rate": 4.99973052523825e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.4629,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 0.95703125,
|
|
"rewards/rejected": -5.5,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.10459511269099775,
|
|
"grad_norm": 6.531824310237248,
|
|
"learning_rate": 4.999680721267631e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.4643,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.10496733729132159,
|
|
"grad_norm": 6.2098151772372665,
|
|
"learning_rate": 4.999626696995509e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.4594,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.10533956189164542,
|
|
"grad_norm": 6.5872029603125615,
|
|
"learning_rate": 4.999568452513096e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.4756,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.10571178649196926,
|
|
"grad_norm": 7.105239867700983,
|
|
"learning_rate": 4.999505987918727e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.4726,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.1060840110922931,
|
|
"grad_norm": 7.548169483272573,
|
|
"learning_rate": 4.999439303317864e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.4815,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -5.625,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.10645623569261692,
|
|
"grad_norm": 7.939773320795078,
|
|
"learning_rate": 4.999368398823093e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -4.0,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.444,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 0.96484375,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.10682846029294076,
|
|
"grad_norm": 7.050604047206517,
|
|
"learning_rate": 4.999293274554124e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.4654,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.890625,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.10720068489326459,
|
|
"grad_norm": 7.817614304146132,
|
|
"learning_rate": 4.999213930637793e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.4577,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.10757290949358843,
|
|
"grad_norm": 7.134591620777526,
|
|
"learning_rate": 4.999130367208059e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.4533,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.10794513409391226,
|
|
"grad_norm": 6.27829079218225,
|
|
"learning_rate": 4.999042584406005e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.4484,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -5.5,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.1083173586942361,
|
|
"grad_norm": 7.389929100160467,
|
|
"learning_rate": 4.998950582379836e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.4542,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.10868958329455994,
|
|
"grad_norm": 5.800758881234425,
|
|
"learning_rate": 4.998854361284885e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -640.0,
|
|
"loss": 0.4683,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 0.98046875,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.10906180789488377,
|
|
"grad_norm": 6.72891021751421,
|
|
"learning_rate": 4.998753921283606e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.4601,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -5.125,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.10943403249520761,
|
|
"grad_norm": 5.891736829425196,
|
|
"learning_rate": 4.998649262545573e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.4532,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.10980625709553145,
|
|
"grad_norm": 6.256548268320358,
|
|
"learning_rate": 4.998540385247487e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.4882,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 0.84375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.11017848169585528,
|
|
"grad_norm": 6.650850428889748,
|
|
"learning_rate": 4.998427289573168e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.4889,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 0.9375,
|
|
"rewards/rejected": -4.75,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.11055070629617911,
|
|
"grad_norm": 5.198683697814229,
|
|
"learning_rate": 4.998309975713561e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.4819,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 0.921875,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.11092293089650294,
|
|
"grad_norm": 5.95366015046444,
|
|
"learning_rate": 4.99818844386673e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.4656,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.11129515549682678,
|
|
"grad_norm": 6.993680918575246,
|
|
"learning_rate": 4.998062694237862e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.454,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.11166738009715062,
|
|
"grad_norm": 5.499983831778576,
|
|
"learning_rate": 4.997932727039265e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.477,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -4.875,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.11203960469747445,
|
|
"grad_norm": 6.275016541282044,
|
|
"learning_rate": 4.997798542490368e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.4388,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 0.9765625,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.11241182929779829,
|
|
"grad_norm": 6.533187769494032,
|
|
"learning_rate": 4.997660140817717e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.4315,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.11278405389812213,
|
|
"grad_norm": 7.138963727779762,
|
|
"learning_rate": 4.997517522254984e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.4499,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.11315627849844596,
|
|
"grad_norm": 7.886282113255406,
|
|
"learning_rate": 4.997370687042956e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.4578,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.1135285030987698,
|
|
"grad_norm": 6.5135054900143246,
|
|
"learning_rate": 4.997219635429538e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.4422,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.11390072769909364,
|
|
"grad_norm": 7.277785076865823,
|
|
"learning_rate": 4.997064367669758e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.4561,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 0.9609375,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.11427295229941747,
|
|
"grad_norm": 5.747350850395194,
|
|
"learning_rate": 4.996904884025761e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.4631,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.11464517689974131,
|
|
"grad_norm": 5.935492702675676,
|
|
"learning_rate": 4.996741184766806e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.4427,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.11501740150006513,
|
|
"grad_norm": 7.403083936549838,
|
|
"learning_rate": 4.996573270169275e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.4474,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.11538962610038897,
|
|
"grad_norm": 6.384910704133043,
|
|
"learning_rate": 4.996401140516663e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4433,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.1157618507007128,
|
|
"grad_norm": 7.451103451639324,
|
|
"learning_rate": 4.996224796099584e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.4448,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -5.625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.11613407530103664,
|
|
"grad_norm": 10.66435519539687,
|
|
"learning_rate": 4.996044237215765e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.451,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.11650629990136048,
|
|
"grad_norm": 8.614773529495455,
|
|
"learning_rate": 4.995859464170051e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.4423,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -5.875,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.11687852450168432,
|
|
"grad_norm": 6.230168528893165,
|
|
"learning_rate": 4.995670477274402e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.4662,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.11725074910200815,
|
|
"grad_norm": 6.630347638446134,
|
|
"learning_rate": 4.995477276847889e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.4494,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 1.0703125,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.11762297370233199,
|
|
"grad_norm": 5.989919817361233,
|
|
"learning_rate": 4.995279863216701e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.4379,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.11799519830265583,
|
|
"grad_norm": 6.854189392325609,
|
|
"learning_rate": 4.995078236714138e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.4494,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.11836742290297966,
|
|
"grad_norm": 6.482345899476014,
|
|
"learning_rate": 4.994872397680615e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.467,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 0.95703125,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.1187396475033035,
|
|
"grad_norm": 10.550904343312228,
|
|
"learning_rate": 4.994662346463655e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.4632,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.11911187210362732,
|
|
"grad_norm": 8.62618156176352,
|
|
"learning_rate": 4.994448083417896e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4431,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 1.2109375,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.11948409670395116,
|
|
"grad_norm": 6.102173546807845,
|
|
"learning_rate": 4.994229608905087e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.4563,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.119856321304275,
|
|
"grad_norm": 6.465859881852491,
|
|
"learning_rate": 4.994006923294085e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.458,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.12022854590459883,
|
|
"grad_norm": 6.7089494535421155,
|
|
"learning_rate": 4.993780026960859e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.4764,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.12060077050492267,
|
|
"grad_norm": 6.321312713360913,
|
|
"learning_rate": 4.993548920288487e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.4619,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.1209729951052465,
|
|
"grad_norm": 6.254706530958953,
|
|
"learning_rate": 4.993313603667152e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.4426,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.12134521970557034,
|
|
"grad_norm": 6.052365903405565,
|
|
"learning_rate": 4.993074077494151e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.4474,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.12171744430589418,
|
|
"grad_norm": 8.853479006674554,
|
|
"learning_rate": 4.992830342173883e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.4225,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.12208966890621802,
|
|
"grad_norm": 10.536193454846627,
|
|
"learning_rate": 4.992582398117854e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.4519,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -6.40625,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.12246189350654185,
|
|
"grad_norm": 6.233893886072476,
|
|
"learning_rate": 4.992330245744679e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4553,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.12283411810686569,
|
|
"grad_norm": 5.553090857036375,
|
|
"learning_rate": 4.992073885480076e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.4666,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 0.8828125,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.12320634270718951,
|
|
"grad_norm": 6.546810988650163,
|
|
"learning_rate": 4.991813317756866e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.4557,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.12357856730751335,
|
|
"grad_norm": 7.251938441351107,
|
|
"learning_rate": 4.991548543014975e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4771,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -6.40625,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.12395079190783719,
|
|
"grad_norm": 6.497208555996562,
|
|
"learning_rate": 4.991279561701434e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.4487,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.12432301650816102,
|
|
"grad_norm": 7.213693539521206,
|
|
"learning_rate": 4.991006374270371e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.4531,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.12469524110848486,
|
|
"grad_norm": 8.12069495771951,
|
|
"learning_rate": 4.990728981183019e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.4554,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.1250674657088087,
|
|
"grad_norm": 5.857658506415699,
|
|
"learning_rate": 4.990447382907713e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.4743,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.12543969030913252,
|
|
"grad_norm": 6.156044660966783,
|
|
"learning_rate": 4.990161579919882e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.4445,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.12581191490945637,
|
|
"grad_norm": 5.802920761503324,
|
|
"learning_rate": 4.98987157270206e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.4552,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.1261841395097802,
|
|
"grad_norm": 7.167183431933801,
|
|
"learning_rate": 4.989577361743875e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.4587,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.12655636411010404,
|
|
"grad_norm": 6.918307973359023,
|
|
"learning_rate": 4.989278947542056e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.415,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.12692858871042786,
|
|
"grad_norm": 7.79049417383309,
|
|
"learning_rate": 4.988976330600426e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.461,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.12730081331075171,
|
|
"grad_norm": 5.812979506475925,
|
|
"learning_rate": 4.988669511429902e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.4646,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.12767303791107554,
|
|
"grad_norm": 5.202303192793724,
|
|
"learning_rate": 4.988358490548501e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.4758,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.1280452625113994,
|
|
"grad_norm": 5.126752700574202,
|
|
"learning_rate": 4.988043268481329e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.4271,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.1284174871117232,
|
|
"grad_norm": 8.729319429305749,
|
|
"learning_rate": 4.987723845760587e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4415,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.12878971171204706,
|
|
"grad_norm": 6.724077120857691,
|
|
"learning_rate": 4.987400222925569e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.4464,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 0.9921875,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.12916193631237088,
|
|
"grad_norm": 5.831736342369829,
|
|
"learning_rate": 4.987072400522658e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.429,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.1295341609126947,
|
|
"grad_norm": 8.807065955668172,
|
|
"learning_rate": 4.986740379105328e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4597,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 0.99609375,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.12990638551301856,
|
|
"grad_norm": 5.358912099415164,
|
|
"learning_rate": 4.986404159234145e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4327,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.13027861011334238,
|
|
"grad_norm": 5.53604733587871,
|
|
"learning_rate": 4.986063741476759e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4544,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.13065083471366623,
|
|
"grad_norm": 6.606069813717992,
|
|
"learning_rate": 4.985719126407912e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.4361,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.13102305931399005,
|
|
"grad_norm": 5.712788808270214,
|
|
"learning_rate": 4.985370314609426e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.4451,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -6.25,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.1313952839143139,
|
|
"grad_norm": 7.268630466589827,
|
|
"learning_rate": 4.985017306670216e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.4517,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.13176750851463773,
|
|
"grad_norm": 5.380960300414308,
|
|
"learning_rate": 4.984660103186278e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4583,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.13213973311496158,
|
|
"grad_norm": 5.495748931798036,
|
|
"learning_rate": 4.984298704760689e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.4337,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.1325119577152854,
|
|
"grad_norm": 7.020317782921593,
|
|
"learning_rate": 4.983933112003615e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4384,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -6.5,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.13288418231560925,
|
|
"grad_norm": 6.2531181232625705,
|
|
"learning_rate": 4.983563325532295e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4424,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.13325640691593307,
|
|
"grad_norm": 6.767773681407881,
|
|
"learning_rate": 4.983189345971056e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.4388,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.1336286315162569,
|
|
"grad_norm": 6.5538001866067725,
|
|
"learning_rate": 4.982811173951301e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4531,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 1.15625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.13400085611658075,
|
|
"grad_norm": 5.932105974565392,
|
|
"learning_rate": 4.982428810111512e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4587,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 0.98828125,
|
|
"rewards/rejected": -6.125,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.13437308071690457,
|
|
"grad_norm": 5.929016937619805,
|
|
"learning_rate": 4.982042255097245e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.4311,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.13474530531722842,
|
|
"grad_norm": 6.9391839889692735,
|
|
"learning_rate": 4.981651509561137e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4472,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.13511752991755224,
|
|
"grad_norm": 7.020061956926156,
|
|
"learning_rate": 4.981256574162897e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4495,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.1354897545178761,
|
|
"grad_norm": 6.6193532141218006,
|
|
"learning_rate": 4.980857449569309e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.4653,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -6.125,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.13586197911819992,
|
|
"grad_norm": 7.247700675584012,
|
|
"learning_rate": 4.98045413645423e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4359,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.13623420371852377,
|
|
"grad_norm": 5.159949207335705,
|
|
"learning_rate": 4.980046635498589e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.4559,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.1366064283188476,
|
|
"grad_norm": 6.062641438365062,
|
|
"learning_rate": 4.979634947390381e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4615,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -6.25,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.13697865291917144,
|
|
"grad_norm": 5.254162964861059,
|
|
"learning_rate": 4.979219072824678e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.445,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.13735087751949526,
|
|
"grad_norm": 5.487312528948721,
|
|
"learning_rate": 4.978799012503614e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.4355,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.13772310211981909,
|
|
"grad_norm": 7.594908714571471,
|
|
"learning_rate": 4.978374767136391e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4589,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.13809532672014294,
|
|
"grad_norm": 7.736311008629555,
|
|
"learning_rate": 4.977946337439281e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4331,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.13846755132046676,
|
|
"grad_norm": 5.947985224213836,
|
|
"learning_rate": 4.977513724135615e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4452,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.1388397759207906,
|
|
"grad_norm": 5.988840734504288,
|
|
"learning_rate": 4.977076927955792e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.455,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.13921200052111443,
|
|
"grad_norm": 5.5971885556830046,
|
|
"learning_rate": 4.976635949637268e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4511,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.13958422512143828,
|
|
"grad_norm": 8.287435415630837,
|
|
"learning_rate": 4.976190789924563e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.4412,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.1399564497217621,
|
|
"grad_norm": 5.790382201491668,
|
|
"learning_rate": 4.975741449569258e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.4597,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 0.9296875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.14032867432208596,
|
|
"grad_norm": 5.124825008551274,
|
|
"learning_rate": 4.975287929329989e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.4558,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.125,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.14070089892240978,
|
|
"grad_norm": 6.493436880612706,
|
|
"learning_rate": 4.974830229972452e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4346,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.14107312352273363,
|
|
"grad_norm": 6.250102477323721,
|
|
"learning_rate": 4.974368352269397e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.4242,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.14144534812305745,
|
|
"grad_norm": 7.074324449573555,
|
|
"learning_rate": 4.973902297000628e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.4489,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.14181757272338127,
|
|
"grad_norm": 5.72103871873261,
|
|
"learning_rate": 4.973432064953004e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4726,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.14218979732370512,
|
|
"grad_norm": 5.227702220552182,
|
|
"learning_rate": 4.972957656920434e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.4352,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.14256202192402895,
|
|
"grad_norm": 5.856768027961113,
|
|
"learning_rate": 4.972479073703879e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4596,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -6.375,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.1429342465243528,
|
|
"grad_norm": 6.732949783918141,
|
|
"learning_rate": 4.971996316111347e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.4298,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.14330647112467662,
|
|
"grad_norm": 7.137832407271159,
|
|
"learning_rate": 4.971509384957899e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4276,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.14367869572500047,
|
|
"grad_norm": 5.67480806034546,
|
|
"learning_rate": 4.971018281065632e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4135,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.1440509203253243,
|
|
"grad_norm": 5.725539657155525,
|
|
"learning_rate": 4.9705230052637e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4571,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -6.25,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.14442314492564814,
|
|
"grad_norm": 6.9494194463470045,
|
|
"learning_rate": 4.970023558388294e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.4452,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -6.0,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.14479536952597197,
|
|
"grad_norm": 5.073889264949155,
|
|
"learning_rate": 4.969519941282647e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.4293,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.0390625,
|
|
"rewards/rejected": -6.0,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.14516759412629582,
|
|
"grad_norm": 22.01836030547002,
|
|
"learning_rate": 4.969012154797034e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4385,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.14553981872661964,
|
|
"grad_norm": 7.421473800191189,
|
|
"learning_rate": 4.96850019978877e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4516,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.1459120433269435,
|
|
"grad_norm": 5.44044171086137,
|
|
"learning_rate": 4.967984077122207e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.4638,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -5.625,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.14628426792726731,
|
|
"grad_norm": 6.467042854573016,
|
|
"learning_rate": 4.967463787668734e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.4384,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.14665649252759114,
|
|
"grad_norm": 5.704803025637172,
|
|
"learning_rate": 4.966939332306773e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.4512,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.147028717127915,
|
|
"grad_norm": 5.4415373853794025,
|
|
"learning_rate": 4.966410711921784e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.4341,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 0.93359375,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.1474009417282388,
|
|
"grad_norm": 6.177926810269377,
|
|
"learning_rate": 4.965877927406252e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4473,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -6.375,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.14777316632856266,
|
|
"grad_norm": 5.659221212517091,
|
|
"learning_rate": 4.965340979659699e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4514,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.14814539092888648,
|
|
"grad_norm": 6.423794139715007,
|
|
"learning_rate": 4.964799869588673e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.4713,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.14851761552921033,
|
|
"grad_norm": 6.008728480613058,
|
|
"learning_rate": 4.964254598106751e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.4413,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.0,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.14888984012953416,
|
|
"grad_norm": 6.688998752516995,
|
|
"learning_rate": 4.96370516613453e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.4574,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 1.0,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.149262064729858,
|
|
"grad_norm": 5.101392987007391,
|
|
"learning_rate": 4.963151574599641e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4559,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -6.375,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.14963428933018183,
|
|
"grad_norm": 6.072334332905822,
|
|
"learning_rate": 4.962593824436731e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.4317,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.15000651393050568,
|
|
"grad_norm": 5.549039175033328,
|
|
"learning_rate": 4.962031916587469e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.4369,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.1503787385308295,
|
|
"grad_norm": 8.33290523579846,
|
|
"learning_rate": 4.961465852000545e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.453,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.15075096313115333,
|
|
"grad_norm": 6.004519762772136,
|
|
"learning_rate": 4.960895631631665e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4448,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.15112318773147718,
|
|
"grad_norm": 6.192962219611233,
|
|
"learning_rate": 4.960321256443555e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.4421,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.151495412331801,
|
|
"grad_norm": 5.612093016508565,
|
|
"learning_rate": 4.959742727405952e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4233,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.15186763693212485,
|
|
"grad_norm": 6.4443742174587095,
|
|
"learning_rate": 4.959160045495607e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.4588,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.15223986153244867,
|
|
"grad_norm": 6.848896537334838,
|
|
"learning_rate": 4.958573211696285e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4566,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -6.25,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.15261208613277252,
|
|
"grad_norm": 6.139629914407962,
|
|
"learning_rate": 4.957982226998757e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4534,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.25,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.15298431073309635,
|
|
"grad_norm": 5.784631101553206,
|
|
"learning_rate": 4.957387092400805e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.4207,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.1533565353334202,
|
|
"grad_norm": 6.765687916525139,
|
|
"learning_rate": 4.956787808907215e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.4489,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.15372875993374402,
|
|
"grad_norm": 6.521721627054844,
|
|
"learning_rate": 4.95618437752978e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4517,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.15410098453406787,
|
|
"grad_norm": 5.413985954025465,
|
|
"learning_rate": 4.955576799287296e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.4504,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.1544732091343917,
|
|
"grad_norm": 5.911781831250542,
|
|
"learning_rate": 4.954965075205556e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.4517,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.15484543373471552,
|
|
"grad_norm": 5.936563366401604,
|
|
"learning_rate": 4.954349206317359e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4484,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -6.625,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.15521765833503937,
|
|
"grad_norm": 6.114561969034227,
|
|
"learning_rate": 4.953729193662498e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4359,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -6.75,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.1555898829353632,
|
|
"grad_norm": 6.144001273005528,
|
|
"learning_rate": 4.953105038287762e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.4461,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.15596210753568704,
|
|
"grad_norm": 4.898030636757637,
|
|
"learning_rate": 4.952476741246937e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.4313,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.15633433213601086,
|
|
"grad_norm": 8.419495217349095,
|
|
"learning_rate": 4.951844303600798e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4361,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -6.625,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.1567065567363347,
|
|
"grad_norm": 7.844881973490722,
|
|
"learning_rate": 4.951207726417113e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4424,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 1.2109375,
|
|
"rewards/rejected": -6.5,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.15707878133665854,
|
|
"grad_norm": 4.96456993515639,
|
|
"learning_rate": 4.950567010770638e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.4335,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -6.25,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.15745100593698239,
|
|
"grad_norm": 6.592505789673692,
|
|
"learning_rate": 4.949922157743116e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.4695,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 1.0703125,
|
|
"rewards/rejected": -6.375,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.1578232305373062,
|
|
"grad_norm": 6.618473760560034,
|
|
"learning_rate": 4.949273168423277e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.4295,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.1328125,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.15819545513763006,
|
|
"grad_norm": 5.821891734797481,
|
|
"learning_rate": 4.948620043906832e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4372,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.15856767973795388,
|
|
"grad_norm": 6.0086092587290105,
|
|
"learning_rate": 4.947962785296475e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.4156,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.1589399043382777,
|
|
"grad_norm": 5.309939126927046,
|
|
"learning_rate": 4.947301393701881e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.408,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.15931212893860155,
|
|
"grad_norm": 8.673672537331084,
|
|
"learning_rate": 4.946635870239699e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4327,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.15968435353892538,
|
|
"grad_norm": 5.252446644861524,
|
|
"learning_rate": 4.945966216033558e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.4371,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.16005657813924923,
|
|
"grad_norm": 6.268235076776519,
|
|
"learning_rate": 4.945292432214059e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4123,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.16042880273957305,
|
|
"grad_norm": 7.476476019382571,
|
|
"learning_rate": 4.944614519918775e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.4605,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.1608010273398969,
|
|
"grad_norm": 5.587260733077475,
|
|
"learning_rate": 4.943932480292251e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.4271,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.16117325194022072,
|
|
"grad_norm": 6.9502310176294815,
|
|
"learning_rate": 4.943246314485999e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.4418,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -6.375,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.16154547654054457,
|
|
"grad_norm": 5.86286884636875,
|
|
"learning_rate": 4.942556023658497e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.4245,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.1619177011408684,
|
|
"grad_norm": 6.997931998512149,
|
|
"learning_rate": 4.941861608975188e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4372,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.16228992574119225,
|
|
"grad_norm": 5.649918984566045,
|
|
"learning_rate": 4.941163071608479e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4421,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.16266215034151607,
|
|
"grad_norm": 5.857293100370255,
|
|
"learning_rate": 4.940460412737733e-07,
|
|
"logits/chosen": -4.1875,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.4316,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.1630343749418399,
|
|
"grad_norm": 5.730729125273545,
|
|
"learning_rate": 4.939753633549277e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.4294,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.16340659954216374,
|
|
"grad_norm": 6.425107180249275,
|
|
"learning_rate": 4.93904273523639e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4478,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.16377882414248757,
|
|
"grad_norm": 4.92933089294134,
|
|
"learning_rate": 4.93832771899931e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.4365,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.16415104874281142,
|
|
"grad_norm": 6.048220715930868,
|
|
"learning_rate": 4.937608586045223e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4345,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.16452327334313524,
|
|
"grad_norm": 5.724400811555856,
|
|
"learning_rate": 4.936885337588266e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4472,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.1648954979434591,
|
|
"grad_norm": 4.948245805297778,
|
|
"learning_rate": 4.936157974849528e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4539,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.1652677225437829,
|
|
"grad_norm": 5.107834343718509,
|
|
"learning_rate": 4.93542649905704e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4305,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.16563994714410676,
|
|
"grad_norm": 5.687476188902605,
|
|
"learning_rate": 4.934690911445782e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4144,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -6.625,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.1660121717444306,
|
|
"grad_norm": 6.4280671159075595,
|
|
"learning_rate": 4.933951213257669e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4558,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.16638439634475444,
|
|
"grad_norm": 5.7637883085404855,
|
|
"learning_rate": 4.933207405741563e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4357,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.16675662094507826,
|
|
"grad_norm": 6.174611383526687,
|
|
"learning_rate": 4.93245949015326e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4445,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -6.75,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.16712884554540208,
|
|
"grad_norm": 5.727568163139461,
|
|
"learning_rate": 4.931707467755495e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4424,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.16750107014572593,
|
|
"grad_norm": 4.96978651041345,
|
|
"learning_rate": 4.930951339817932e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4443,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.16787329474604976,
|
|
"grad_norm": 6.606143113936588,
|
|
"learning_rate": 4.93019110761717e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.4461,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.1682455193463736,
|
|
"grad_norm": 5.280551169780135,
|
|
"learning_rate": 4.929426772436738e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4594,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.16861774394669743,
|
|
"grad_norm": 5.903286901824118,
|
|
"learning_rate": 4.928658335567088e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.4336,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.16898996854702128,
|
|
"grad_norm": 5.421764382713266,
|
|
"learning_rate": 4.927885798305603e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.432,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -7.25,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.1693621931473451,
|
|
"grad_norm": 5.463647088242241,
|
|
"learning_rate": 4.927109161956584e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.4502,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 1.0,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.16973441774766895,
|
|
"grad_norm": 7.0277650426833596,
|
|
"learning_rate": 4.926328427831254e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4375,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.17010664234799278,
|
|
"grad_norm": 5.163554543735777,
|
|
"learning_rate": 4.925543597247756e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4278,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.17047886694831663,
|
|
"grad_norm": 5.520417597352931,
|
|
"learning_rate": 4.924754671531145e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.17085109154864045,
|
|
"grad_norm": 5.835278895528116,
|
|
"learning_rate": 4.923961652013396e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.4337,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.17122331614896427,
|
|
"grad_norm": 6.289577139251418,
|
|
"learning_rate": 4.923164540033392e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.4395,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -6.25,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.17159554074928812,
|
|
"grad_norm": 5.124573188721107,
|
|
"learning_rate": 4.922363336936926e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.17196776534961195,
|
|
"grad_norm": 5.0454831945737,
|
|
"learning_rate": 4.921558044076696e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4346,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.1723399899499358,
|
|
"grad_norm": 5.347507423472996,
|
|
"learning_rate": 4.920748662812309e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4591,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.17271221455025962,
|
|
"grad_norm": 5.887680168469224,
|
|
"learning_rate": 4.919935194510274e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.4363,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.17308443915058347,
|
|
"grad_norm": 5.01156054278049,
|
|
"learning_rate": 4.919117640543996e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4401,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.1734566637509073,
|
|
"grad_norm": 5.984393551781648,
|
|
"learning_rate": 4.918296002293782e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.4405,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 0.9296875,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.17382888835123114,
|
|
"grad_norm": 5.486756525242989,
|
|
"learning_rate": 4.917470281146836e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4384,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.17420111295155497,
|
|
"grad_norm": 6.836449830380156,
|
|
"learning_rate": 4.916640478497249e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4185,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.17457333755187882,
|
|
"grad_norm": 6.681647488020188,
|
|
"learning_rate": 4.91580659574601e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4412,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.17494556215220264,
|
|
"grad_norm": 6.294748260060898,
|
|
"learning_rate": 4.914968634300993e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4471,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.17531778675252646,
|
|
"grad_norm": 5.943100595893998,
|
|
"learning_rate": 4.914126595576957e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4246,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.1756900113528503,
|
|
"grad_norm": 6.101951406933382,
|
|
"learning_rate": 4.913280480995547e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4535,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.17606223595317413,
|
|
"grad_norm": 8.302285652731438,
|
|
"learning_rate": 4.912430291985291e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4365,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.17643446055349798,
|
|
"grad_norm": 6.811876284942206,
|
|
"learning_rate": 4.911576029981591e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4267,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.1768066851538218,
|
|
"grad_norm": 6.426324871968156,
|
|
"learning_rate": 4.91071769642673e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.4309,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.17717890975414566,
|
|
"grad_norm": 5.803178246026337,
|
|
"learning_rate": 4.909855292769863e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.4506,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.17755113435446948,
|
|
"grad_norm": 6.950723716770103,
|
|
"learning_rate": 4.908988820467018e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4353,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.17792335895479333,
|
|
"grad_norm": 5.9514924809140535,
|
|
"learning_rate": 4.908118280981091e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4409,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.17829558355511715,
|
|
"grad_norm": 5.418539622734764,
|
|
"learning_rate": 4.907243675781847e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.4251,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.178667808155441,
|
|
"grad_norm": 6.290411507285996,
|
|
"learning_rate": 4.90636500634591e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4211,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.17904003275576483,
|
|
"grad_norm": 6.944629306948202,
|
|
"learning_rate": 4.905482274156773e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4403,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.17941225735608865,
|
|
"grad_norm": 6.05476571168117,
|
|
"learning_rate": 4.904595480704784e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.452,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.1797844819564125,
|
|
"grad_norm": 6.155027965189672,
|
|
"learning_rate": 4.903704627487148e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4216,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.18015670655673632,
|
|
"grad_norm": 5.568715078016802,
|
|
"learning_rate": 4.902809716007923e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4478,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.18052893115706017,
|
|
"grad_norm": 6.21218353743708,
|
|
"learning_rate": 4.901910747778023e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.4445,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.75,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.180901155757384,
|
|
"grad_norm": 6.46285181147374,
|
|
"learning_rate": 4.901007724315209e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.419,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.18127338035770785,
|
|
"grad_norm": 5.760644621772445,
|
|
"learning_rate": 4.900100647144085e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4443,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.18164560495803167,
|
|
"grad_norm": 6.191332890605878,
|
|
"learning_rate": 4.899189517796105e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4357,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.18201782955835552,
|
|
"grad_norm": 5.754330079438478,
|
|
"learning_rate": 4.898274337809562e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4243,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.18239005415867934,
|
|
"grad_norm": 6.10788076623001,
|
|
"learning_rate": 4.897355108729585e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4273,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.1827622787590032,
|
|
"grad_norm": 8.019138920938525,
|
|
"learning_rate": 4.896431832108143e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4498,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.375,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.18313450335932702,
|
|
"grad_norm": 5.790248495304021,
|
|
"learning_rate": 4.895504509504038e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.18350672795965087,
|
|
"grad_norm": 5.005123399748186,
|
|
"learning_rate": 4.894573142482902e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4314,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.1838789525599747,
|
|
"grad_norm": 5.863132919555019,
|
|
"learning_rate": 4.893637732617196e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4293,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.1842511771602985,
|
|
"grad_norm": 5.565479595150576,
|
|
"learning_rate": 4.892698281486206e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4177,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.18462340176062236,
|
|
"grad_norm": 5.740628607292125,
|
|
"learning_rate": 4.89175479067604e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4349,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.18499562636094619,
|
|
"grad_norm": 5.952183752271304,
|
|
"learning_rate": 4.890807261779631e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4411,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.18536785096127004,
|
|
"grad_norm": 7.133701595887453,
|
|
"learning_rate": 4.889855696396722e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4398,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.18574007556159386,
|
|
"grad_norm": 6.923918556888372,
|
|
"learning_rate": 4.88890009613388e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4227,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.1861123001619177,
|
|
"grad_norm": 6.291949952274157,
|
|
"learning_rate": 4.887940462604475e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4437,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.18648452476224153,
|
|
"grad_norm": 5.824934736201254,
|
|
"learning_rate": 4.886976797428694e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4433,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.2109375,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"epoch": 0.18685674936256538,
|
|
"grad_norm": 6.7809786186754994,
|
|
"learning_rate": 4.886009102233528e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4446,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"epoch": 0.1872289739628892,
|
|
"grad_norm": 6.572483137865605,
|
|
"learning_rate": 4.88503737865277e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4388,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.0,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"epoch": 0.18760119856321306,
|
|
"grad_norm": 5.875787555846081,
|
|
"learning_rate": 4.884061628327018e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4408,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"epoch": 0.18797342316353688,
|
|
"grad_norm": 6.63099560911551,
|
|
"learning_rate": 4.883081852903665e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.3993,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"epoch": 0.1883456477638607,
|
|
"grad_norm": 6.192004094990947,
|
|
"learning_rate": 4.882098054036901e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4449,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"epoch": 0.18871787236418455,
|
|
"grad_norm": 5.449841513167424,
|
|
"learning_rate": 4.881110233387711e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"epoch": 0.18909009696450838,
|
|
"grad_norm": 6.611642424312423,
|
|
"learning_rate": 4.880118392623869e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.4422,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -7.125,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"epoch": 0.18946232156483223,
|
|
"grad_norm": 6.406689262062489,
|
|
"learning_rate": 4.879122533419933e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4273,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"epoch": 0.18983454616515605,
|
|
"grad_norm": 6.870439202846314,
|
|
"learning_rate": 4.87812265745725e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4398,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 1.2109375,
|
|
"rewards/rejected": -6.875,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"epoch": 0.1902067707654799,
|
|
"grad_norm": 5.810037669339324,
|
|
"learning_rate": 4.877118766423945e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4212,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"epoch": 0.19057899536580372,
|
|
"grad_norm": 6.766938365906897,
|
|
"learning_rate": 4.876110862014926e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4369,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"epoch": 0.19095121996612757,
|
|
"grad_norm": 6.761489994304975,
|
|
"learning_rate": 4.875098945931873e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4173,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"epoch": 0.1913234445664514,
|
|
"grad_norm": 7.698292169233494,
|
|
"learning_rate": 4.874083019883242e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4515,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"epoch": 0.19169566916677525,
|
|
"grad_norm": 6.269374749190229,
|
|
"learning_rate": 4.873063085584256e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4388,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"epoch": 0.19206789376709907,
|
|
"grad_norm": 7.319111042537406,
|
|
"learning_rate": 4.872039144756907e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.4154,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"epoch": 0.1924401183674229,
|
|
"grad_norm": 5.771849169408466,
|
|
"learning_rate": 4.871011199129953e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.4296,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"epoch": 0.19281234296774674,
|
|
"grad_norm": 8.346007061944526,
|
|
"learning_rate": 4.869979250438911e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4258,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"epoch": 0.19318456756807056,
|
|
"grad_norm": 5.882045339240376,
|
|
"learning_rate": 4.868943300426057e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4616,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.0,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"epoch": 0.19355679216839441,
|
|
"grad_norm": 8.188921380148814,
|
|
"learning_rate": 4.867903350840423e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.4367,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.19392901676871824,
|
|
"grad_norm": 6.594589444122374,
|
|
"learning_rate": 4.866859403437795e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"epoch": 0.1943012413690421,
|
|
"grad_norm": 6.496047738910236,
|
|
"learning_rate": 4.865811459980705e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"epoch": 0.1946734659693659,
|
|
"grad_norm": 6.510238689502243,
|
|
"learning_rate": 4.864759522238435e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4582,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.0,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"epoch": 0.19504569056968976,
|
|
"grad_norm": 4.970398430245081,
|
|
"learning_rate": 4.86370359198701e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4327,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"epoch": 0.19541791517001358,
|
|
"grad_norm": 5.626167819252166,
|
|
"learning_rate": 4.862643671009193e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4347,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"epoch": 0.19579013977033743,
|
|
"grad_norm": 5.719147146022044,
|
|
"learning_rate": 4.861579761094491e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4216,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"epoch": 0.19616236437066126,
|
|
"grad_norm": 5.982037492273309,
|
|
"learning_rate": 4.860511864039139e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4495,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"epoch": 0.19653458897098508,
|
|
"grad_norm": 5.7757974423024905,
|
|
"learning_rate": 4.859439981646106e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"epoch": 0.19690681357130893,
|
|
"grad_norm": 5.935875763124951,
|
|
"learning_rate": 4.858364115725091e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.0,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"epoch": 0.19727903817163275,
|
|
"grad_norm": 7.208331646547231,
|
|
"learning_rate": 4.857284268092516e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4362,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"epoch": 0.1976512627719566,
|
|
"grad_norm": 5.132504170247751,
|
|
"learning_rate": 4.856200440571529e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.413,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"epoch": 0.19802348737228043,
|
|
"grad_norm": 7.103692250248847,
|
|
"learning_rate": 4.855112634991994e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.4255,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"epoch": 0.19839571197260428,
|
|
"grad_norm": 6.060147946883416,
|
|
"learning_rate": 4.854020853190492e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4336,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"epoch": 0.1987679365729281,
|
|
"grad_norm": 8.111034449123615,
|
|
"learning_rate": 4.85292509701032e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4222,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"epoch": 0.19914016117325195,
|
|
"grad_norm": 6.418688758365402,
|
|
"learning_rate": 4.85182536830148e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4177,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"epoch": 0.19951238577357577,
|
|
"grad_norm": 8.499175173716342,
|
|
"learning_rate": 4.850721668920684e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.4399,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"epoch": 0.19988461037389962,
|
|
"grad_norm": 5.648775417303448,
|
|
"learning_rate": 4.84961400073135e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.4235,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"epoch": 0.20025683497422345,
|
|
"grad_norm": 6.147892375161948,
|
|
"learning_rate": 4.848502365603593e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4263,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"epoch": 0.20062905957454727,
|
|
"grad_norm": 6.305737555389114,
|
|
"learning_rate": 4.847386765414227e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4134,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"epoch": 0.20100128417487112,
|
|
"grad_norm": 6.756698949690817,
|
|
"learning_rate": 4.84626720204676e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4111,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.20137350877519494,
|
|
"grad_norm": 6.583951650314223,
|
|
"learning_rate": 4.845143677391392e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4008,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"epoch": 0.2017457333755188,
|
|
"grad_norm": 8.150984712049954,
|
|
"learning_rate": 4.84401619334501e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4487,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"epoch": 0.20211795797584262,
|
|
"grad_norm": 11.44870084333935,
|
|
"learning_rate": 4.842884751811185e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4272,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"epoch": 0.20249018257616647,
|
|
"grad_norm": 6.4106474615628875,
|
|
"learning_rate": 4.841749354700172e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4384,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"epoch": 0.2028624071764903,
|
|
"grad_norm": 7.794629882532868,
|
|
"learning_rate": 4.840610003928902e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.423,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"epoch": 0.20323463177681414,
|
|
"grad_norm": 6.853660297097755,
|
|
"learning_rate": 4.839466701420985e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4598,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"epoch": 0.20360685637713796,
|
|
"grad_norm": 5.5885088165224435,
|
|
"learning_rate": 4.838319449106697e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4248,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"epoch": 0.2039790809774618,
|
|
"grad_norm": 6.098976321073987,
|
|
"learning_rate": 4.837168248922986e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"epoch": 0.20435130557778564,
|
|
"grad_norm": 6.9691567272920345,
|
|
"learning_rate": 4.836013102813467e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.416,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"epoch": 0.20472353017810946,
|
|
"grad_norm": 6.987026770248282,
|
|
"learning_rate": 4.834854012728412e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4467,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 0.2050957547784333,
|
|
"grad_norm": 7.332931226582443,
|
|
"learning_rate": 4.833690980624758e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4421,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.15625,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"epoch": 0.20546797937875713,
|
|
"grad_norm": 5.574105363176493,
|
|
"learning_rate": 4.832524008466091e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4302,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"epoch": 0.20584020397908098,
|
|
"grad_norm": 5.818091335733473,
|
|
"learning_rate": 4.831353098222655e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4418,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"epoch": 0.2062124285794048,
|
|
"grad_norm": 6.318937634194189,
|
|
"learning_rate": 4.83017825187134e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4136,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"epoch": 0.20658465317972866,
|
|
"grad_norm": 6.362560807781102,
|
|
"learning_rate": 4.828999471395679e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4171,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"epoch": 0.20695687778005248,
|
|
"grad_norm": 6.837123822999831,
|
|
"learning_rate": 4.827816758785853e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4168,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"epoch": 0.20732910238037633,
|
|
"grad_norm": 7.634964550191584,
|
|
"learning_rate": 4.826630116038677e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4264,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"epoch": 0.20770132698070015,
|
|
"grad_norm": 6.576149738258397,
|
|
"learning_rate": 4.825439545157603e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4539,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"epoch": 0.208073551581024,
|
|
"grad_norm": 5.342907939993414,
|
|
"learning_rate": 4.824245048152715e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4379,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"epoch": 0.20844577618134782,
|
|
"grad_norm": 5.576272430665703,
|
|
"learning_rate": 4.823046627040725e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4479,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.20881800078167165,
|
|
"grad_norm": 6.132824090079501,
|
|
"learning_rate": 4.821844283844972e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4256,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"epoch": 0.2091902253819955,
|
|
"grad_norm": 6.744167887351889,
|
|
"learning_rate": 4.820638020595414e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4362,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"epoch": 0.20956244998231932,
|
|
"grad_norm": 6.68286891786841,
|
|
"learning_rate": 4.819427839328632e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.4326,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"epoch": 0.20993467458264317,
|
|
"grad_norm": 5.760935231480335,
|
|
"learning_rate": 4.818213742087815e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4208,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"epoch": 0.210306899182967,
|
|
"grad_norm": 6.890033542530431,
|
|
"learning_rate": 4.81699573092277e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4133,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"epoch": 0.21067912378329084,
|
|
"grad_norm": 6.569652591342706,
|
|
"learning_rate": 4.815773807889907e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4139,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"epoch": 0.21105134838361467,
|
|
"grad_norm": 5.474514412647331,
|
|
"learning_rate": 4.814547975052244e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4429,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"epoch": 0.21142357298393852,
|
|
"grad_norm": 8.429225542364062,
|
|
"learning_rate": 4.813318234479401e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4303,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"epoch": 0.21179579758426234,
|
|
"grad_norm": 7.135542873605128,
|
|
"learning_rate": 4.812084588247592e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4211,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"epoch": 0.2121680221845862,
|
|
"grad_norm": 6.292181622463565,
|
|
"learning_rate": 4.810847038439626e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4401,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"epoch": 0.21254024678491001,
|
|
"grad_norm": 5.891486428128075,
|
|
"learning_rate": 4.809605587144904e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4401,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"epoch": 0.21291247138523384,
|
|
"grad_norm": 5.959544850834762,
|
|
"learning_rate": 4.808360236459412e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4149,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"epoch": 0.2132846959855577,
|
|
"grad_norm": 5.303988732591514,
|
|
"learning_rate": 4.807110988485721e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4346,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"epoch": 0.2136569205858815,
|
|
"grad_norm": 5.241951241473354,
|
|
"learning_rate": 4.805857845332983e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"epoch": 0.21402914518620536,
|
|
"grad_norm": 5.724771685599554,
|
|
"learning_rate": 4.804600809116925e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4182,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"epoch": 0.21440136978652918,
|
|
"grad_norm": 7.018939661902944,
|
|
"learning_rate": 4.803339881959845e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4365,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"epoch": 0.21477359438685303,
|
|
"grad_norm": 7.1140689469871115,
|
|
"learning_rate": 4.802075065990613e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.4382,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"epoch": 0.21514581898717686,
|
|
"grad_norm": 6.4150311160162135,
|
|
"learning_rate": 4.800806363344663e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4447,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"epoch": 0.2155180435875007,
|
|
"grad_norm": 5.750678354752301,
|
|
"learning_rate": 4.799533776163993e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4239,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"epoch": 0.21589026818782453,
|
|
"grad_norm": 6.160905396738596,
|
|
"learning_rate": 4.798257306597156e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.405,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.078125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.21626249278814838,
|
|
"grad_norm": 6.255992324791328,
|
|
"learning_rate": 4.796976956799264e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4387,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"epoch": 0.2166347173884722,
|
|
"grad_norm": 6.358479027139798,
|
|
"learning_rate": 4.795692728931977e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.438,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"epoch": 0.21700694198879603,
|
|
"grad_norm": 5.584716791155845,
|
|
"learning_rate": 4.794404625163503e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4163,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"epoch": 0.21737916658911988,
|
|
"grad_norm": 5.02637272530766,
|
|
"learning_rate": 4.793112647668594e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4232,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"epoch": 0.2177513911894437,
|
|
"grad_norm": 6.243005960274981,
|
|
"learning_rate": 4.791816798628544e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4275,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"epoch": 0.21812361578976755,
|
|
"grad_norm": 6.334112755551554,
|
|
"learning_rate": 4.790517080231179e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4553,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"epoch": 0.21849584039009137,
|
|
"grad_norm": 8.624590744980026,
|
|
"learning_rate": 4.789213494670864e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4376,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"epoch": 0.21886806499041522,
|
|
"grad_norm": 5.302347633998562,
|
|
"learning_rate": 4.787906044148489e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.4204,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"epoch": 0.21924028959073905,
|
|
"grad_norm": 5.760136212014773,
|
|
"learning_rate": 4.786594730871467e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4121,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"epoch": 0.2196125141910629,
|
|
"grad_norm": 6.194875548358758,
|
|
"learning_rate": 4.785279557053739e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4523,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"epoch": 0.21998473879138672,
|
|
"grad_norm": 5.9746838607629,
|
|
"learning_rate": 4.78396052491576e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4174,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"epoch": 0.22035696339171057,
|
|
"grad_norm": 5.004404488177059,
|
|
"learning_rate": 4.782637636684499e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"epoch": 0.2207291879920344,
|
|
"grad_norm": 6.276216370797568,
|
|
"learning_rate": 4.781310894593437e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4375,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"epoch": 0.22110141259235822,
|
|
"grad_norm": 6.831699485958554,
|
|
"learning_rate": 4.779980300882559e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.407,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"epoch": 0.22147363719268207,
|
|
"grad_norm": 6.5151009940251,
|
|
"learning_rate": 4.778645857798357e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.4257,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"epoch": 0.2218458617930059,
|
|
"grad_norm": 6.108704670003317,
|
|
"learning_rate": 4.777307567593818e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4277,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -7.375,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"epoch": 0.22221808639332974,
|
|
"grad_norm": 6.840284114854557,
|
|
"learning_rate": 4.775965432528426e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4125,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"epoch": 0.22259031099365356,
|
|
"grad_norm": 6.289011208186768,
|
|
"learning_rate": 4.774619454868156e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4215,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"epoch": 0.2229625355939774,
|
|
"grad_norm": 5.619923035928777,
|
|
"learning_rate": 4.773269636885471e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"epoch": 0.22333476019430124,
|
|
"grad_norm": 5.705051528328636,
|
|
"learning_rate": 4.771915980859318e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4239,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.22370698479462509,
|
|
"grad_norm": 8.73830036477818,
|
|
"learning_rate": 4.770558489075124e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4103,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"epoch": 0.2240792093949489,
|
|
"grad_norm": 7.699953594593835,
|
|
"learning_rate": 4.76919716382479e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4203,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"epoch": 0.22445143399527276,
|
|
"grad_norm": 5.641880745608248,
|
|
"learning_rate": 4.7678320074066925e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"epoch": 0.22482365859559658,
|
|
"grad_norm": 7.766493501204703,
|
|
"learning_rate": 4.766463022125673e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4192,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"epoch": 0.22519588319592043,
|
|
"grad_norm": 11.90030598484292,
|
|
"learning_rate": 4.765090210293039e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4338,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"epoch": 0.22556810779624425,
|
|
"grad_norm": 5.3912321345938645,
|
|
"learning_rate": 4.76371357422656e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.413,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"epoch": 0.22594033239656808,
|
|
"grad_norm": 6.247487437915781,
|
|
"learning_rate": 4.7623331162504585e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"epoch": 0.22631255699689193,
|
|
"grad_norm": 4.912612975965821,
|
|
"learning_rate": 4.7609488386954137e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4312,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"epoch": 0.22668478159721575,
|
|
"grad_norm": 6.238925386472296,
|
|
"learning_rate": 4.759560743898551e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4039,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"epoch": 0.2270570061975396,
|
|
"grad_norm": 5.1875835889428465,
|
|
"learning_rate": 4.758168834203439e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.3976,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"epoch": 0.22742923079786342,
|
|
"grad_norm": 8.485382086718937,
|
|
"learning_rate": 4.7567731119600916e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.414,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"epoch": 0.22780145539818727,
|
|
"grad_norm": 6.1986468421974354,
|
|
"learning_rate": 4.755373579524957e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4438,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"epoch": 0.2281736799985111,
|
|
"grad_norm": 5.09697993938089,
|
|
"learning_rate": 4.753970239260916e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4568,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"epoch": 0.22854590459883495,
|
|
"grad_norm": 5.216105600386573,
|
|
"learning_rate": 4.752563093537279e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4172,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.375,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"epoch": 0.22891812919915877,
|
|
"grad_norm": 5.771226303646994,
|
|
"learning_rate": 4.751152144729781e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4235,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"epoch": 0.22929035379948262,
|
|
"grad_norm": 6.631240571601773,
|
|
"learning_rate": 4.749737395220578e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4056,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"epoch": 0.22966257839980644,
|
|
"grad_norm": 5.943297152893245,
|
|
"learning_rate": 4.748318847398242e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4205,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"epoch": 0.23003480300013027,
|
|
"grad_norm": 5.159509186113557,
|
|
"learning_rate": 4.746896503657759e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4269,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"epoch": 0.23040702760045412,
|
|
"grad_norm": 5.278332364946635,
|
|
"learning_rate": 4.745470366400525e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4082,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"epoch": 0.23077925220077794,
|
|
"grad_norm": 6.3291603399206995,
|
|
"learning_rate": 4.744040438034338e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4303,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.2311514768011018,
|
|
"grad_norm": 5.6835786857173805,
|
|
"learning_rate": 4.7426067209733977e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.4292,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"epoch": 0.2315237014014256,
|
|
"grad_norm": 6.795692015670665,
|
|
"learning_rate": 4.7411692176383013e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4084,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"epoch": 0.23189592600174946,
|
|
"grad_norm": 5.928900460558585,
|
|
"learning_rate": 4.739727930456037e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4134,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"epoch": 0.2322681506020733,
|
|
"grad_norm": 5.694796850977235,
|
|
"learning_rate": 4.738282861859982e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4298,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"epoch": 0.23264037520239714,
|
|
"grad_norm": 5.276815425170437,
|
|
"learning_rate": 4.7368340142898983e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.4339,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"epoch": 0.23301259980272096,
|
|
"grad_norm": 6.11808955978257,
|
|
"learning_rate": 4.7353813901919283e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4197,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.375,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"epoch": 0.2333848244030448,
|
|
"grad_norm": 6.585944369333626,
|
|
"learning_rate": 4.7339249920185885e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4155,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"epoch": 0.23375704900336863,
|
|
"grad_norm": 5.665815747348609,
|
|
"learning_rate": 4.73246482222877e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"epoch": 0.23412927360369246,
|
|
"grad_norm": 8.25782804113197,
|
|
"learning_rate": 4.73100088328773e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4289,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"epoch": 0.2345014982040163,
|
|
"grad_norm": 5.321909412094669,
|
|
"learning_rate": 4.72953317766709e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4366,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"epoch": 0.23487372280434013,
|
|
"grad_norm": 5.675167377702157,
|
|
"learning_rate": 4.7280617078448294e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4287,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"epoch": 0.23524594740466398,
|
|
"grad_norm": 5.980308610466193,
|
|
"learning_rate": 4.726586476305285e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4276,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"epoch": 0.2356181720049878,
|
|
"grad_norm": 5.240960005134948,
|
|
"learning_rate": 4.725107485539143e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4393,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"epoch": 0.23599039660531165,
|
|
"grad_norm": 5.477851232251246,
|
|
"learning_rate": 4.723624738043438e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4293,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"epoch": 0.23636262120563548,
|
|
"grad_norm": 5.841123757011323,
|
|
"learning_rate": 4.7221382363215447e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4363,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"epoch": 0.23673484580595933,
|
|
"grad_norm": 5.710257314719626,
|
|
"learning_rate": 4.72064798288318e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4062,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"epoch": 0.23710707040628315,
|
|
"grad_norm": 6.599006254069624,
|
|
"learning_rate": 4.7191539802443906e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4179,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"epoch": 0.237479295006607,
|
|
"grad_norm": 5.709786268939416,
|
|
"learning_rate": 4.717656230927557e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4261,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"epoch": 0.23785151960693082,
|
|
"grad_norm": 6.7867426232994275,
|
|
"learning_rate": 4.7161547374613817e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4168,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"epoch": 0.23822374420725465,
|
|
"grad_norm": 6.53386853099891,
|
|
"learning_rate": 4.714649502380893e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4198,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.2385959688075785,
|
|
"grad_norm": 6.391274328229158,
|
|
"learning_rate": 4.7131405282274315e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4478,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"epoch": 0.23896819340790232,
|
|
"grad_norm": 5.107772466037207,
|
|
"learning_rate": 4.7116278175486546e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4437,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"epoch": 0.23934041800822617,
|
|
"grad_norm": 4.827325580734079,
|
|
"learning_rate": 4.7101113728985253e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4199,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"epoch": 0.23971264260855,
|
|
"grad_norm": 5.7011857021652945,
|
|
"learning_rate": 4.7085911968373135e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"epoch": 0.24008486720887384,
|
|
"grad_norm": 5.461782220817337,
|
|
"learning_rate": 4.7070672919315856e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4192,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"epoch": 0.24045709180919767,
|
|
"grad_norm": 6.381709528302522,
|
|
"learning_rate": 4.705539660754207e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4285,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"epoch": 0.24082931640952152,
|
|
"grad_norm": 5.25881298503053,
|
|
"learning_rate": 4.704008305884332e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4421,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"epoch": 0.24120154100984534,
|
|
"grad_norm": 6.1454630595308295,
|
|
"learning_rate": 4.7024732299074023e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4231,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"epoch": 0.2415737656101692,
|
|
"grad_norm": 7.608828921397016,
|
|
"learning_rate": 4.7009344354151424e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"epoch": 0.241945990210493,
|
|
"grad_norm": 7.361323116966944,
|
|
"learning_rate": 4.6993919250055557e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.439,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 0.24231821481081683,
|
|
"grad_norm": 6.519730606394934,
|
|
"learning_rate": 4.6978457012829174e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"epoch": 0.24269043941114068,
|
|
"grad_norm": 6.195135220830553,
|
|
"learning_rate": 4.6962957668577736e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4291,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"epoch": 0.2430626640114645,
|
|
"grad_norm": 5.530513448208236,
|
|
"learning_rate": 4.694742124346934e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4383,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"epoch": 0.24343488861178836,
|
|
"grad_norm": 6.563109720471082,
|
|
"learning_rate": 4.6931847763734703e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4279,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"epoch": 0.24380711321211218,
|
|
"grad_norm": 5.592632811272104,
|
|
"learning_rate": 4.6916237255667093e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4265,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"epoch": 0.24417933781243603,
|
|
"grad_norm": 5.819942978536498,
|
|
"learning_rate": 4.6900589745622294e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4306,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"epoch": 0.24455156241275985,
|
|
"grad_norm": 6.01567534022582,
|
|
"learning_rate": 4.688490526001856e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4141,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"epoch": 0.2449237870130837,
|
|
"grad_norm": 8.31094093560855,
|
|
"learning_rate": 4.6869183825336587e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.455,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"epoch": 0.24529601161340753,
|
|
"grad_norm": 5.487859257431511,
|
|
"learning_rate": 4.685342546811943e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4281,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"epoch": 0.24566823621373138,
|
|
"grad_norm": 6.186567569700576,
|
|
"learning_rate": 4.6837630214972514e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4476,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.2460404608140552,
|
|
"grad_norm": 5.841327794112783,
|
|
"learning_rate": 4.6821798092563514e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"epoch": 0.24641268541437902,
|
|
"grad_norm": 6.629758179486839,
|
|
"learning_rate": 4.680592912762238e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"epoch": 0.24678491001470287,
|
|
"grad_norm": 8.464610313608668,
|
|
"learning_rate": 4.6790023346941274e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4185,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"epoch": 0.2471571346150267,
|
|
"grad_norm": 7.712268152224804,
|
|
"learning_rate": 4.677408077737449e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4408,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"epoch": 0.24752935921535055,
|
|
"grad_norm": 5.4645917339902415,
|
|
"learning_rate": 4.6758101445838457e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4343,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"epoch": 0.24790158381567437,
|
|
"grad_norm": 7.6134801516815065,
|
|
"learning_rate": 4.6742085379311645e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4069,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"epoch": 0.24827380841599822,
|
|
"grad_norm": 5.942050379509909,
|
|
"learning_rate": 4.6726032604834566e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"epoch": 0.24864603301632204,
|
|
"grad_norm": 5.822387109912939,
|
|
"learning_rate": 4.67099431495097e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.427,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"epoch": 0.2490182576166459,
|
|
"grad_norm": 8.346740221535079,
|
|
"learning_rate": 4.669381704050146e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4247,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"epoch": 0.24939048221696972,
|
|
"grad_norm": 5.849544355776524,
|
|
"learning_rate": 4.6677654305036136e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.439,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"epoch": 0.24976270681729357,
|
|
"grad_norm": 5.3230508705704915,
|
|
"learning_rate": 4.666145497040186e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4151,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"epoch": 0.2501349314176174,
|
|
"grad_norm": 6.019226326104658,
|
|
"learning_rate": 4.664521906394856e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4254,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"epoch": 0.2505071560179412,
|
|
"grad_norm": 9.714162090429475,
|
|
"learning_rate": 4.662894661308789e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.4502,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"epoch": 0.25087938061826504,
|
|
"grad_norm": 5.802439874398323,
|
|
"learning_rate": 4.6612637645293243e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.44,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.375,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"epoch": 0.2512516052185889,
|
|
"grad_norm": 6.056722990489438,
|
|
"learning_rate": 4.659629218809963e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4106,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"epoch": 0.25162382981891274,
|
|
"grad_norm": 5.567097289524385,
|
|
"learning_rate": 4.657991026910366e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4099,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"epoch": 0.25199605441923656,
|
|
"grad_norm": 6.667974417121959,
|
|
"learning_rate": 4.656349191596355e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4031,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"epoch": 0.2523682790195604,
|
|
"grad_norm": 7.8323725233471935,
|
|
"learning_rate": 4.6547037156398976e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.4167,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"epoch": 0.25274050361988426,
|
|
"grad_norm": 6.077973638002682,
|
|
"learning_rate": 4.653054601819112e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4232,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"epoch": 0.2531127282202081,
|
|
"grad_norm": 6.566988113849512,
|
|
"learning_rate": 4.651401852918256e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4461,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.2534849528205319,
|
|
"grad_norm": 5.971675664298436,
|
|
"learning_rate": 4.6497454717277253e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4189,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.375,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"epoch": 0.25385717742085573,
|
|
"grad_norm": 5.721771079114988,
|
|
"learning_rate": 4.648085461044049e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.427,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"epoch": 0.2542294020211796,
|
|
"grad_norm": 5.818955687434478,
|
|
"learning_rate": 4.646421823669883e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4371,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"epoch": 0.25460162662150343,
|
|
"grad_norm": 5.935606451539939,
|
|
"learning_rate": 4.644754562414006e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -7.25,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"epoch": 0.25497385122182725,
|
|
"grad_norm": 4.920274001439463,
|
|
"learning_rate": 4.6430836800913167e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.3982,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"epoch": 0.2553460758221511,
|
|
"grad_norm": 7.030292287431365,
|
|
"learning_rate": 4.6414091795228247e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4015,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"epoch": 0.2557183004224749,
|
|
"grad_norm": 6.75799958648115,
|
|
"learning_rate": 4.6397310635356514e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4338,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"epoch": 0.2560905250227988,
|
|
"grad_norm": 5.712416690771472,
|
|
"learning_rate": 4.63804933496302e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4282,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"epoch": 0.2564627496231226,
|
|
"grad_norm": 5.805375772360747,
|
|
"learning_rate": 4.6363639966442535e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4298,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"epoch": 0.2568349742234464,
|
|
"grad_norm": 8.145232038439593,
|
|
"learning_rate": 4.634675051424771e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4341,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"epoch": 0.25720719882377024,
|
|
"grad_norm": 5.980559160504537,
|
|
"learning_rate": 4.632982502156078e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4239,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"epoch": 0.2575794234240941,
|
|
"grad_norm": 6.764275312649953,
|
|
"learning_rate": 4.6312863516957686e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4192,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"epoch": 0.25795164802441795,
|
|
"grad_norm": 5.7918311969978955,
|
|
"learning_rate": 4.629586602907514e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4516,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"epoch": 0.25832387262474177,
|
|
"grad_norm": 5.931704932586348,
|
|
"learning_rate": 4.627883258661061e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"epoch": 0.2586960972250656,
|
|
"grad_norm": 6.120552916217351,
|
|
"learning_rate": 4.626176321832229e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4186,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"epoch": 0.2590683218253894,
|
|
"grad_norm": 6.18407137107997,
|
|
"learning_rate": 4.6244657953029005e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4157,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"epoch": 0.2594405464257133,
|
|
"grad_norm": 6.3884031653947435,
|
|
"learning_rate": 4.622751681961019e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4237,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"epoch": 0.2598127710260371,
|
|
"grad_norm": 5.758457196120104,
|
|
"learning_rate": 4.621033984700585e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4061,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"epoch": 0.26018499562636094,
|
|
"grad_norm": 5.825420787649575,
|
|
"learning_rate": 4.6193127064216486e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"epoch": 0.26055722022668476,
|
|
"grad_norm": 5.613717574869501,
|
|
"learning_rate": 4.6175878500303054e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4008,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.26092944482700864,
|
|
"grad_norm": 7.354862413572649,
|
|
"learning_rate": 4.615859418438695e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"epoch": 0.26130166942733246,
|
|
"grad_norm": 5.708966665313372,
|
|
"learning_rate": 4.6141274145649876e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4245,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"epoch": 0.2616738940276563,
|
|
"grad_norm": 5.64831204522663,
|
|
"learning_rate": 4.612391841333392e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"epoch": 0.2620461186279801,
|
|
"grad_norm": 5.614310797181401,
|
|
"learning_rate": 4.6106527016741377e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4126,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"epoch": 0.262418343228304,
|
|
"grad_norm": 7.4644644596942005,
|
|
"learning_rate": 4.608909998523476e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4261,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"epoch": 0.2627905678286278,
|
|
"grad_norm": 6.188756503030487,
|
|
"learning_rate": 4.607163734823678e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"epoch": 0.26316279242895163,
|
|
"grad_norm": 5.7489193983212585,
|
|
"learning_rate": 4.605413913523022e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.421,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"epoch": 0.26353501702927545,
|
|
"grad_norm": 6.267474216073482,
|
|
"learning_rate": 4.603660537575796e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4193,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"epoch": 0.2639072416295993,
|
|
"grad_norm": 7.087242445477736,
|
|
"learning_rate": 4.601903609942287e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.417,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"epoch": 0.26427946622992315,
|
|
"grad_norm": 6.325037036344175,
|
|
"learning_rate": 4.600143133588781e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.3953,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"epoch": 0.264651690830247,
|
|
"grad_norm": 7.109490718254207,
|
|
"learning_rate": 4.598379111487552e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4288,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"epoch": 0.2650239154305708,
|
|
"grad_norm": 6.612417698596598,
|
|
"learning_rate": 4.5966115466168645e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4257,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"epoch": 0.2653961400308946,
|
|
"grad_norm": 7.0194486136726875,
|
|
"learning_rate": 4.594840441960961e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4227,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"epoch": 0.2657683646312185,
|
|
"grad_norm": 5.401212137042596,
|
|
"learning_rate": 4.593065800510062e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.425,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"epoch": 0.2661405892315423,
|
|
"grad_norm": 5.232057431338868,
|
|
"learning_rate": 4.5912876252603585e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4113,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"epoch": 0.26651281383186615,
|
|
"grad_norm": 6.052061286777042,
|
|
"learning_rate": 4.5895059192140095e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4289,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"epoch": 0.26688503843218997,
|
|
"grad_norm": 6.368459265997135,
|
|
"learning_rate": 4.587720685379132e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4355,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"epoch": 0.2672572630325138,
|
|
"grad_norm": 6.558928363280608,
|
|
"learning_rate": 4.5859319267698025e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4352,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"epoch": 0.26762948763283767,
|
|
"grad_norm": 5.963429130104855,
|
|
"learning_rate": 4.584139646406047e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"epoch": 0.2680017122331615,
|
|
"grad_norm": 5.280230908834761,
|
|
"learning_rate": 4.5823438473138353e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4213,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.2683739368334853,
|
|
"grad_norm": 5.847075646593318,
|
|
"learning_rate": 4.5805445325250826e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4083,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"epoch": 0.26874616143380914,
|
|
"grad_norm": 8.482225886648067,
|
|
"learning_rate": 4.578741705077636e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4446,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"epoch": 0.269118386034133,
|
|
"grad_norm": 5.399312421088563,
|
|
"learning_rate": 4.5769353680152735e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4355,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"epoch": 0.26949061063445684,
|
|
"grad_norm": 5.441411541551882,
|
|
"learning_rate": 4.575125524387701e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4255,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"epoch": 0.26986283523478066,
|
|
"grad_norm": 5.607984311565896,
|
|
"learning_rate": 4.573312177250543e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4139,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"epoch": 0.2702350598351045,
|
|
"grad_norm": 6.039940537323914,
|
|
"learning_rate": 4.571495329665338e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4401,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"epoch": 0.27060728443542836,
|
|
"grad_norm": 5.790152845082641,
|
|
"learning_rate": 4.5696749846995365e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4374,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"epoch": 0.2709795090357522,
|
|
"grad_norm": 6.1954310400298835,
|
|
"learning_rate": 4.5678511454264924e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4117,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"epoch": 0.271351733636076,
|
|
"grad_norm": 6.029955581819493,
|
|
"learning_rate": 4.566023814925459e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4281,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"epoch": 0.27172395823639983,
|
|
"grad_norm": 6.191821880326933,
|
|
"learning_rate": 4.5641929962815863e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4421,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"epoch": 0.27209618283672365,
|
|
"grad_norm": 5.089637557125682,
|
|
"learning_rate": 4.56235869258591e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.4345,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"epoch": 0.27246840743704753,
|
|
"grad_norm": 5.611604046597495,
|
|
"learning_rate": 4.5605209069353525e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.3938,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"epoch": 0.27284063203737136,
|
|
"grad_norm": 5.955603263327238,
|
|
"learning_rate": 4.5586796424327135e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4405,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"epoch": 0.2732128566376952,
|
|
"grad_norm": 6.377692630412435,
|
|
"learning_rate": 4.5568349021866666e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4138,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"epoch": 0.273585081238019,
|
|
"grad_norm": 6.7136703934171775,
|
|
"learning_rate": 4.554986689311754e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4345,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"epoch": 0.2739573058383429,
|
|
"grad_norm": 6.01075360902772,
|
|
"learning_rate": 4.553135006928379e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4349,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"epoch": 0.2743295304386667,
|
|
"grad_norm": 7.956713507898213,
|
|
"learning_rate": 4.551279858162806e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"epoch": 0.2747017550389905,
|
|
"grad_norm": 6.412918379270744,
|
|
"learning_rate": 4.549421246147149e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4217,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"epoch": 0.27507397963931435,
|
|
"grad_norm": 5.9595415971913175,
|
|
"learning_rate": 4.547559174019369e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"epoch": 0.27544620423963817,
|
|
"grad_norm": 6.131025186944429,
|
|
"learning_rate": 4.5456936449232715e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4158,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.27581842883996205,
|
|
"grad_norm": 5.7655881844347245,
|
|
"learning_rate": 4.543824662008496e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4091,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"epoch": 0.27619065344028587,
|
|
"grad_norm": 6.058077064969408,
|
|
"learning_rate": 4.541952228430514e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4294,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"epoch": 0.2765628780406097,
|
|
"grad_norm": 8.553526571063612,
|
|
"learning_rate": 4.540076347350623e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"epoch": 0.2769351026409335,
|
|
"grad_norm": 11.03091735656846,
|
|
"learning_rate": 4.538197021935941e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"epoch": 0.2773073272412574,
|
|
"grad_norm": 6.61976796266326,
|
|
"learning_rate": 4.5363142553594014e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.4055,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"epoch": 0.2776795518415812,
|
|
"grad_norm": 6.8849285588537885,
|
|
"learning_rate": 4.534428050799747e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4169,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"epoch": 0.27805177644190504,
|
|
"grad_norm": 5.805460007962187,
|
|
"learning_rate": 4.5325384114415254e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"epoch": 0.27842400104222886,
|
|
"grad_norm": 7.638616985649138,
|
|
"learning_rate": 4.530645340475083e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"epoch": 0.27879622564255274,
|
|
"grad_norm": 6.657490517120866,
|
|
"learning_rate": 4.52874884109656e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4189,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"epoch": 0.27916845024287656,
|
|
"grad_norm": 5.88141696203444,
|
|
"learning_rate": 4.5268489165078855e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4326,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 0.2795406748432004,
|
|
"grad_norm": 6.871904732080597,
|
|
"learning_rate": 4.5249455699167706e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.4171,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"epoch": 0.2799128994435242,
|
|
"grad_norm": 6.612468422522421,
|
|
"learning_rate": 4.523038804536704e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.4151,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"epoch": 0.28028512404384803,
|
|
"grad_norm": 6.992999825567599,
|
|
"learning_rate": 4.521128623586947e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4343,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"epoch": 0.2806573486441719,
|
|
"grad_norm": 6.188508914849754,
|
|
"learning_rate": 4.519215030292527e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4374,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"epoch": 0.28102957324449573,
|
|
"grad_norm": 5.878247526319351,
|
|
"learning_rate": 4.517298027884234e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.43,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.5,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"epoch": 0.28140179784481956,
|
|
"grad_norm": 6.09574265694494,
|
|
"learning_rate": 4.5153776195986113e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.4354,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.046875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"epoch": 0.2817740224451434,
|
|
"grad_norm": 5.579202560668251,
|
|
"learning_rate": 4.513453808677955e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.3867,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"epoch": 0.28214624704546726,
|
|
"grad_norm": 6.414315148440845,
|
|
"learning_rate": 4.5115265983703036e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"epoch": 0.2825184716457911,
|
|
"grad_norm": 5.37839396130642,
|
|
"learning_rate": 4.5095959919294366e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4132,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"epoch": 0.2828906962461149,
|
|
"grad_norm": 8.683317484337856,
|
|
"learning_rate": 4.5076619926148673e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4209,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.2832629208464387,
|
|
"grad_norm": 6.779098935746247,
|
|
"learning_rate": 4.5057246036918357e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4081,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"epoch": 0.28363514544676255,
|
|
"grad_norm": 5.788869742179201,
|
|
"learning_rate": 4.503783828431306e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4149,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"epoch": 0.2840073700470864,
|
|
"grad_norm": 8.676483883656678,
|
|
"learning_rate": 4.50183967010996e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"epoch": 0.28437959464741025,
|
|
"grad_norm": 5.407369224295794,
|
|
"learning_rate": 4.49989213201019e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.4227,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"epoch": 0.2847518192477341,
|
|
"grad_norm": 5.496551436680179,
|
|
"learning_rate": 4.497941217420095e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.4007,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"epoch": 0.2851240438480579,
|
|
"grad_norm": 6.3776460810407665,
|
|
"learning_rate": 4.495986929633476e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4334,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"epoch": 0.2854962684483818,
|
|
"grad_norm": 7.728309008830767,
|
|
"learning_rate": 4.494029271949826e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4077,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"epoch": 0.2858684930487056,
|
|
"grad_norm": 6.387763824439387,
|
|
"learning_rate": 4.492068247674331e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4597,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"epoch": 0.2862407176490294,
|
|
"grad_norm": 7.421524518905735,
|
|
"learning_rate": 4.490103860117858e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4338,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"epoch": 0.28661294224935324,
|
|
"grad_norm": 5.350457321169909,
|
|
"learning_rate": 4.4881361125969546e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4127,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"epoch": 0.2869851668496771,
|
|
"grad_norm": 5.835040909626544,
|
|
"learning_rate": 4.48616500843384e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4233,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"epoch": 0.28735739145000094,
|
|
"grad_norm": 6.0248412291469435,
|
|
"learning_rate": 4.4841905509564e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -4.0625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"epoch": 0.28772961605032477,
|
|
"grad_norm": 5.837526251953547,
|
|
"learning_rate": 4.4822127434981845e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"epoch": 0.2881018406506486,
|
|
"grad_norm": 6.5140837455567695,
|
|
"learning_rate": 4.4802315893983957e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4104,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"epoch": 0.2884740652509724,
|
|
"grad_norm": 5.619844171623824,
|
|
"learning_rate": 4.4782470920018875e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4229,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"epoch": 0.2888462898512963,
|
|
"grad_norm": 5.647042840024987,
|
|
"learning_rate": 4.4762592546591617e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4305,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"epoch": 0.2892185144516201,
|
|
"grad_norm": 5.718864330463644,
|
|
"learning_rate": 4.4742680807263524e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4264,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"epoch": 0.28959073905194394,
|
|
"grad_norm": 6.107401923218653,
|
|
"learning_rate": 4.4722735735652327e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4453,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"epoch": 0.28996296365226776,
|
|
"grad_norm": 5.147574770400471,
|
|
"learning_rate": 4.4702757365432007e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4153,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"epoch": 0.29033518825259164,
|
|
"grad_norm": 5.677173879103329,
|
|
"learning_rate": 4.468274573033278e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4251,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.29070741285291546,
|
|
"grad_norm": 5.766540373890742,
|
|
"learning_rate": 4.4662700864141e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4182,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"epoch": 0.2910796374532393,
|
|
"grad_norm": 6.9286497803910265,
|
|
"learning_rate": 4.4642622800699155e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4193,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"epoch": 0.2914518620535631,
|
|
"grad_norm": 5.93524793533176,
|
|
"learning_rate": 4.4622511573905754e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -7.75,
|
|
"step": 7830
|
|
},
|
|
{
|
|
"epoch": 0.291824086653887,
|
|
"grad_norm": 5.557305101607596,
|
|
"learning_rate": 4.460236721771531e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4139,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"epoch": 0.2921963112542108,
|
|
"grad_norm": 8.193446093915714,
|
|
"learning_rate": 4.458218976613828e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.425,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 7850
|
|
},
|
|
{
|
|
"epoch": 0.29256853585453463,
|
|
"grad_norm": 6.122405971175107,
|
|
"learning_rate": 4.456197925324098e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.426,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"epoch": 0.29294076045485845,
|
|
"grad_norm": 5.771768562447178,
|
|
"learning_rate": 4.4541735713145546e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4111,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 7870
|
|
},
|
|
{
|
|
"epoch": 0.2933129850551823,
|
|
"grad_norm": 5.6990150415497185,
|
|
"learning_rate": 4.4521459180029884e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4269,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"epoch": 0.29368520965550615,
|
|
"grad_norm": 5.681072560503333,
|
|
"learning_rate": 4.45011496881276e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4146,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 7890
|
|
},
|
|
{
|
|
"epoch": 0.29405743425583,
|
|
"grad_norm": 5.931316505507531,
|
|
"learning_rate": 4.4480807271727954e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4124,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"epoch": 0.2944296588561538,
|
|
"grad_norm": 6.555282071925908,
|
|
"learning_rate": 4.446043196517577e-07,
|
|
"logits/chosen": -4.15625,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4446,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 7910
|
|
},
|
|
{
|
|
"epoch": 0.2948018834564776,
|
|
"grad_norm": 6.99953419167626,
|
|
"learning_rate": 4.444002380287143e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4173,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"epoch": 0.2951741080568015,
|
|
"grad_norm": 5.614691661723048,
|
|
"learning_rate": 4.441958281927075e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4182,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 7930
|
|
},
|
|
{
|
|
"epoch": 0.2955463326571253,
|
|
"grad_norm": 6.5137816481943345,
|
|
"learning_rate": 4.4399109048885006e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4264,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"epoch": 0.29591855725744914,
|
|
"grad_norm": 6.137050836358027,
|
|
"learning_rate": 4.437860252628081e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.3919,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7950
|
|
},
|
|
{
|
|
"epoch": 0.29629078185777297,
|
|
"grad_norm": 6.602019028812518,
|
|
"learning_rate": 4.435806328608004e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4004,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"epoch": 0.2966630064580968,
|
|
"grad_norm": 6.383267054442638,
|
|
"learning_rate": 4.4337491362959854e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4067,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7970
|
|
},
|
|
{
|
|
"epoch": 0.29703523105842067,
|
|
"grad_norm": 7.246108384770167,
|
|
"learning_rate": 4.4316886791652584e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"epoch": 0.2974074556587445,
|
|
"grad_norm": 7.119303505897672,
|
|
"learning_rate": 4.429624960694566e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4212,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 7990
|
|
},
|
|
{
|
|
"epoch": 0.2977796802590683,
|
|
"grad_norm": 7.49237550667767,
|
|
"learning_rate": 4.42755798436816e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4071,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 0.29815190485939214,
|
|
"grad_norm": 5.940738972935634,
|
|
"learning_rate": 4.42548775367579e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4075,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8010
|
|
},
|
|
{
|
|
"epoch": 0.298524129459716,
|
|
"grad_norm": 6.9285959946410625,
|
|
"learning_rate": 4.4234142721127026e-07,
|
|
"logits/chosen": -4.1875,
|
|
"logits/rejected": -4.03125,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"epoch": 0.29889635406003984,
|
|
"grad_norm": 5.13595531237585,
|
|
"learning_rate": 4.4213375431796316e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4053,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8030
|
|
},
|
|
{
|
|
"epoch": 0.29926857866036366,
|
|
"grad_norm": 5.936162615807526,
|
|
"learning_rate": 4.419257570382793e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4263,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"epoch": 0.2996408032606875,
|
|
"grad_norm": 6.419512204504135,
|
|
"learning_rate": 4.4171743572338813e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4599,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8050
|
|
},
|
|
{
|
|
"epoch": 0.30001302786101136,
|
|
"grad_norm": 5.8554478476790255,
|
|
"learning_rate": 4.4150879072500604e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"epoch": 0.3003852524613352,
|
|
"grad_norm": 5.931214591098493,
|
|
"learning_rate": 4.4129982239539594e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4166,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8070
|
|
},
|
|
{
|
|
"epoch": 0.300757477061659,
|
|
"grad_norm": 5.611931945679536,
|
|
"learning_rate": 4.410905310873665e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"epoch": 0.30112970166198283,
|
|
"grad_norm": 5.574444546827678,
|
|
"learning_rate": 4.40880917154272e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8090
|
|
},
|
|
{
|
|
"epoch": 0.30150192626230665,
|
|
"grad_norm": 7.186521731548357,
|
|
"learning_rate": 4.4067098095001113e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4158,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"epoch": 0.30187415086263053,
|
|
"grad_norm": 7.481605924985874,
|
|
"learning_rate": 4.4046072282902687e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4124,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 8110
|
|
},
|
|
{
|
|
"epoch": 0.30224637546295435,
|
|
"grad_norm": 6.3696058520993155,
|
|
"learning_rate": 4.402501431463055e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.4327,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"epoch": 0.3026186000632782,
|
|
"grad_norm": 7.618684490235694,
|
|
"learning_rate": 4.4003924225737643e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 8130
|
|
},
|
|
{
|
|
"epoch": 0.302990824663602,
|
|
"grad_norm": 5.540607847130943,
|
|
"learning_rate": 4.3982802051831127e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"epoch": 0.3033630492639259,
|
|
"grad_norm": 6.278444493282474,
|
|
"learning_rate": 4.396164782857232e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.416,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8150
|
|
},
|
|
{
|
|
"epoch": 0.3037352738642497,
|
|
"grad_norm": 5.626823879718762,
|
|
"learning_rate": 4.3940461591676683e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"epoch": 0.3041074984645735,
|
|
"grad_norm": 6.691950006306523,
|
|
"learning_rate": 4.391924337691368e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.3957,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 8170
|
|
},
|
|
{
|
|
"epoch": 0.30447972306489735,
|
|
"grad_norm": 6.886288410461233,
|
|
"learning_rate": 4.3897993220106825e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4083,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"epoch": 0.30485194766522117,
|
|
"grad_norm": 7.030614261123283,
|
|
"learning_rate": 4.3876711157133506e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4104,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 8190
|
|
},
|
|
{
|
|
"epoch": 0.30522417226554505,
|
|
"grad_norm": 5.28685154258976,
|
|
"learning_rate": 4.385539722392501e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"epoch": 0.30559639686586887,
|
|
"grad_norm": 6.1831163140416,
|
|
"learning_rate": 4.3834051456466414e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4147,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 8210
|
|
},
|
|
{
|
|
"epoch": 0.3059686214661927,
|
|
"grad_norm": 6.8383476647506205,
|
|
"learning_rate": 4.381267389079656e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4041,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"epoch": 0.3063408460665165,
|
|
"grad_norm": 8.026714295802059,
|
|
"learning_rate": 4.379126456300796e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4079,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 8230
|
|
},
|
|
{
|
|
"epoch": 0.3067130706668404,
|
|
"grad_norm": 5.558690125341562,
|
|
"learning_rate": 4.3769823509246753e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4347,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"epoch": 0.3070852952671642,
|
|
"grad_norm": 5.815296242571299,
|
|
"learning_rate": 4.374835076571265e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4121,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -7.625,
|
|
"step": 8250
|
|
},
|
|
{
|
|
"epoch": 0.30745751986748804,
|
|
"grad_norm": 7.325015608535012,
|
|
"learning_rate": 4.3726846368658874e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4266,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"epoch": 0.30782974446781186,
|
|
"grad_norm": 8.500449221632431,
|
|
"learning_rate": 4.370531035439206e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4356,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 8270
|
|
},
|
|
{
|
|
"epoch": 0.30820196906813574,
|
|
"grad_norm": 5.416677126597945,
|
|
"learning_rate": 4.3683742759272255e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4142,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"epoch": 0.30857419366845956,
|
|
"grad_norm": 5.580238074384531,
|
|
"learning_rate": 4.36621436197128e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4044,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8290
|
|
},
|
|
{
|
|
"epoch": 0.3089464182687834,
|
|
"grad_norm": 6.409107181809076,
|
|
"learning_rate": 4.364051297218031e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4133,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"epoch": 0.3093186428691072,
|
|
"grad_norm": 6.8985678741594745,
|
|
"learning_rate": 4.361885085319459e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4349,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 8310
|
|
},
|
|
{
|
|
"epoch": 0.30969086746943103,
|
|
"grad_norm": 5.638952892806989,
|
|
"learning_rate": 4.359715729932858e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.4192,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"epoch": 0.3100630920697549,
|
|
"grad_norm": 7.246221958164084,
|
|
"learning_rate": 4.357543234720829e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4257,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 8330
|
|
},
|
|
{
|
|
"epoch": 0.31043531667007873,
|
|
"grad_norm": 6.092445253017867,
|
|
"learning_rate": 4.355367603351275e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.407,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"epoch": 0.31080754127040255,
|
|
"grad_norm": 5.426934262227511,
|
|
"learning_rate": 4.353188839497393e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4023,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 8350
|
|
},
|
|
{
|
|
"epoch": 0.3111797658707264,
|
|
"grad_norm": 7.317631447645299,
|
|
"learning_rate": 4.3510069468376687e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4053,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"epoch": 0.31155199047105026,
|
|
"grad_norm": 6.952255929461985,
|
|
"learning_rate": 4.34882192905587e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.3986,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8370
|
|
},
|
|
{
|
|
"epoch": 0.3119242150713741,
|
|
"grad_norm": 5.970769551386224,
|
|
"learning_rate": 4.3466337898410435e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"epoch": 0.3122964396716979,
|
|
"grad_norm": 6.849106146083491,
|
|
"learning_rate": 4.3444425328875013e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4166,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 8390
|
|
},
|
|
{
|
|
"epoch": 0.3126686642720217,
|
|
"grad_norm": 6.2180243526623356,
|
|
"learning_rate": 4.3422481618948236e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4049,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"epoch": 0.31304088887234555,
|
|
"grad_norm": 6.082562983185604,
|
|
"learning_rate": 4.340050680567846e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4232,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8410
|
|
},
|
|
{
|
|
"epoch": 0.3134131134726694,
|
|
"grad_norm": 7.44201695017634,
|
|
"learning_rate": 4.337850092616656e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"epoch": 0.31378533807299325,
|
|
"grad_norm": 5.9405189431510745,
|
|
"learning_rate": 4.3356464017565856e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8430
|
|
},
|
|
{
|
|
"epoch": 0.31415756267331707,
|
|
"grad_norm": 5.924181931732808,
|
|
"learning_rate": 4.333439611708206e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3944,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"epoch": 0.3145297872736409,
|
|
"grad_norm": 6.4765620173851115,
|
|
"learning_rate": 4.3312297261973206e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8450
|
|
},
|
|
{
|
|
"epoch": 0.31490201187396477,
|
|
"grad_norm": 5.671122846877211,
|
|
"learning_rate": 4.32901674895496e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.4111,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"epoch": 0.3152742364742886,
|
|
"grad_norm": 6.657463307251734,
|
|
"learning_rate": 4.326800683717373e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4179,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8470
|
|
},
|
|
{
|
|
"epoch": 0.3156464610746124,
|
|
"grad_norm": 5.75342405182809,
|
|
"learning_rate": 4.324581534226023e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4293,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"epoch": 0.31601868567493624,
|
|
"grad_norm": 6.836465597847896,
|
|
"learning_rate": 4.32235930422758e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.3967,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8490
|
|
},
|
|
{
|
|
"epoch": 0.3163909102752601,
|
|
"grad_norm": 6.651365659289901,
|
|
"learning_rate": 4.3201339974739165e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4233,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"epoch": 0.31676313487558394,
|
|
"grad_norm": 7.359793337944927,
|
|
"learning_rate": 4.3179056177220976e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4236,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8510
|
|
},
|
|
{
|
|
"epoch": 0.31713535947590776,
|
|
"grad_norm": 7.067853932231795,
|
|
"learning_rate": 4.3156741687343776e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4426,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"epoch": 0.3175075840762316,
|
|
"grad_norm": 5.996014220524351,
|
|
"learning_rate": 4.313439654278194e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4108,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8530
|
|
},
|
|
{
|
|
"epoch": 0.3178798086765554,
|
|
"grad_norm": 5.238535042979966,
|
|
"learning_rate": 4.311202078126156e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"epoch": 0.3182520332768793,
|
|
"grad_norm": 5.012702460857521,
|
|
"learning_rate": 4.308961444056046e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4208,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 8550
|
|
},
|
|
{
|
|
"epoch": 0.3186242578772031,
|
|
"grad_norm": 11.305688686391719,
|
|
"learning_rate": 4.306717755850808e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4059,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"epoch": 0.31899648247752693,
|
|
"grad_norm": 5.295829030657762,
|
|
"learning_rate": 4.304471017298542e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4067,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8570
|
|
},
|
|
{
|
|
"epoch": 0.31936870707785076,
|
|
"grad_norm": 5.913689328599416,
|
|
"learning_rate": 4.302221232192497e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4283,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"epoch": 0.31974093167817463,
|
|
"grad_norm": 5.784266238781056,
|
|
"learning_rate": 4.2999684043310667e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.3987,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8590
|
|
},
|
|
{
|
|
"epoch": 0.32011315627849846,
|
|
"grad_norm": 5.464018820974973,
|
|
"learning_rate": 4.297712537517784e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3807,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"epoch": 0.3204853808788223,
|
|
"grad_norm": 7.923595559439052,
|
|
"learning_rate": 4.295453635561308e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4234,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 8610
|
|
},
|
|
{
|
|
"epoch": 0.3208576054791461,
|
|
"grad_norm": 5.0265425003094455,
|
|
"learning_rate": 4.293191702275426e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.399,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"epoch": 0.3212298300794699,
|
|
"grad_norm": 6.43312477412012,
|
|
"learning_rate": 4.290926741479043e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 8630
|
|
},
|
|
{
|
|
"epoch": 0.3216020546797938,
|
|
"grad_norm": 6.491899681684533,
|
|
"learning_rate": 4.288658756996172e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4087,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"epoch": 0.3219742792801176,
|
|
"grad_norm": 6.426639427626988,
|
|
"learning_rate": 4.2863877526559344e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4083,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8650
|
|
},
|
|
{
|
|
"epoch": 0.32234650388044145,
|
|
"grad_norm": 6.168189668870453,
|
|
"learning_rate": 4.2841137322925493e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"epoch": 0.32271872848076527,
|
|
"grad_norm": 6.127930009387289,
|
|
"learning_rate": 4.281836699745327e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 8670
|
|
},
|
|
{
|
|
"epoch": 0.32309095308108915,
|
|
"grad_norm": 6.702853514547645,
|
|
"learning_rate": 4.279556658858665e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4368,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"epoch": 0.32346317768141297,
|
|
"grad_norm": 5.723774547547514,
|
|
"learning_rate": 4.2772736134820385e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.402,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8690
|
|
},
|
|
{
|
|
"epoch": 0.3238354022817368,
|
|
"grad_norm": 5.980546634229164,
|
|
"learning_rate": 4.2749875674699954e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"epoch": 0.3242076268820606,
|
|
"grad_norm": 6.209159869839144,
|
|
"learning_rate": 4.2726985246821507e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3925,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8710
|
|
},
|
|
{
|
|
"epoch": 0.3245798514823845,
|
|
"grad_norm": 6.267291076001385,
|
|
"learning_rate": 4.270406488983177e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4185,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"epoch": 0.3249520760827083,
|
|
"grad_norm": 7.578543422656428,
|
|
"learning_rate": 4.268111464242803e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4124,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8730
|
|
},
|
|
{
|
|
"epoch": 0.32532430068303214,
|
|
"grad_norm": 7.375260990815802,
|
|
"learning_rate": 4.2658134543358e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.388,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"epoch": 0.32569652528335596,
|
|
"grad_norm": 6.525551561252898,
|
|
"learning_rate": 4.2635124631419827e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8750
|
|
},
|
|
{
|
|
"epoch": 0.3260687498836798,
|
|
"grad_norm": 7.26237339228251,
|
|
"learning_rate": 4.261208494546198e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4243,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"epoch": 0.32644097448400367,
|
|
"grad_norm": 5.842136712955005,
|
|
"learning_rate": 4.2589015524383187e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4155,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8770
|
|
},
|
|
{
|
|
"epoch": 0.3268131990843275,
|
|
"grad_norm": 6.234556903970917,
|
|
"learning_rate": 4.2565916407132393e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4092,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"epoch": 0.3271854236846513,
|
|
"grad_norm": 6.349897441965635,
|
|
"learning_rate": 4.254278763270867e-07,
|
|
"logits/chosen": -4.21875,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4305,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8790
|
|
},
|
|
{
|
|
"epoch": 0.32755764828497513,
|
|
"grad_norm": 6.7361433127902215,
|
|
"learning_rate": 4.251962924016117e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"epoch": 0.327929872885299,
|
|
"grad_norm": 7.804511550663237,
|
|
"learning_rate": 4.2496441268589047e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4002,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8810
|
|
},
|
|
{
|
|
"epoch": 0.32830209748562283,
|
|
"grad_norm": 6.655558491999033,
|
|
"learning_rate": 4.2473223757141386e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.393,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"epoch": 0.32867432208594666,
|
|
"grad_norm": 6.864977853309613,
|
|
"learning_rate": 4.2449976745017157e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.41,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8830
|
|
},
|
|
{
|
|
"epoch": 0.3290465466862705,
|
|
"grad_norm": 6.5844947633559,
|
|
"learning_rate": 4.2426700271465134e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -4.0625,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"epoch": 0.32941877128659436,
|
|
"grad_norm": 6.819245813260431,
|
|
"learning_rate": 4.240339437578383e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4102,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 8850
|
|
},
|
|
{
|
|
"epoch": 0.3297909958869182,
|
|
"grad_norm": 5.76134641555927,
|
|
"learning_rate": 4.238005909732144e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"epoch": 0.330163220487242,
|
|
"grad_norm": 5.874282433157733,
|
|
"learning_rate": 4.235669447547574e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4129,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8870
|
|
},
|
|
{
|
|
"epoch": 0.3305354450875658,
|
|
"grad_norm": 5.972758576716602,
|
|
"learning_rate": 4.2333300549694085e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4176,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"epoch": 0.33090766968788965,
|
|
"grad_norm": 5.067748708967267,
|
|
"learning_rate": 4.2309877359473277e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.4214,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 8890
|
|
},
|
|
{
|
|
"epoch": 0.33127989428821353,
|
|
"grad_norm": 6.122715243608545,
|
|
"learning_rate": 4.2286424944359547e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4249,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"epoch": 0.33165211888853735,
|
|
"grad_norm": 6.74866694741633,
|
|
"learning_rate": 4.2262943343948445e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4089,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8910
|
|
},
|
|
{
|
|
"epoch": 0.3320243434888612,
|
|
"grad_norm": 5.646707898613816,
|
|
"learning_rate": 4.223943259788481e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.3928,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"epoch": 0.332396568089185,
|
|
"grad_norm": 5.631200896660625,
|
|
"learning_rate": 4.22158927458627e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4221,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8930
|
|
},
|
|
{
|
|
"epoch": 0.3327687926895089,
|
|
"grad_norm": 6.053408040794421,
|
|
"learning_rate": 4.219232382762528e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4045,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"epoch": 0.3331410172898327,
|
|
"grad_norm": 6.323042450967391,
|
|
"learning_rate": 4.2168725882964825e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4261,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 8950
|
|
},
|
|
{
|
|
"epoch": 0.3335132418901565,
|
|
"grad_norm": 5.828864435359478,
|
|
"learning_rate": 4.2145098951722584e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4267,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"epoch": 0.33388546649048034,
|
|
"grad_norm": 5.70859261213098,
|
|
"learning_rate": 4.2121443073788775e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3929,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8970
|
|
},
|
|
{
|
|
"epoch": 0.33425769109080417,
|
|
"grad_norm": 6.088627683559499,
|
|
"learning_rate": 4.209775828910247e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4091,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"epoch": 0.33462991569112804,
|
|
"grad_norm": 5.693198970274243,
|
|
"learning_rate": 4.207404463765154e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8990
|
|
},
|
|
{
|
|
"epoch": 0.33500214029145187,
|
|
"grad_norm": 6.190797373208319,
|
|
"learning_rate": 4.205030215947261e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4231,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 0.3353743648917757,
|
|
"grad_norm": 7.9204455765894775,
|
|
"learning_rate": 4.202653089465097e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.4138,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 9010
|
|
},
|
|
{
|
|
"epoch": 0.3357465894920995,
|
|
"grad_norm": 5.967744946826578,
|
|
"learning_rate": 4.2002730883320493e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -4.0,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4155,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"epoch": 0.3361188140924234,
|
|
"grad_norm": 5.457856710766668,
|
|
"learning_rate": 4.1978902165663616e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4214,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9030
|
|
},
|
|
{
|
|
"epoch": 0.3364910386927472,
|
|
"grad_norm": 5.747191781439095,
|
|
"learning_rate": 4.1955044781911215e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4429,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"epoch": 0.33686326329307104,
|
|
"grad_norm": 7.912218403652893,
|
|
"learning_rate": 4.193115877234258e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4238,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 9050
|
|
},
|
|
{
|
|
"epoch": 0.33723548789339486,
|
|
"grad_norm": 5.910070642286067,
|
|
"learning_rate": 4.1907244177285326e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4219,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"epoch": 0.33760771249371874,
|
|
"grad_norm": 6.0530639916655335,
|
|
"learning_rate": 4.188330103711533e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9070
|
|
},
|
|
{
|
|
"epoch": 0.33797993709404256,
|
|
"grad_norm": 7.750779225230822,
|
|
"learning_rate": 4.185932939225666e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4266,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"epoch": 0.3383521616943664,
|
|
"grad_norm": 6.525563977104109,
|
|
"learning_rate": 4.1835329283181506e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.402,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9090
|
|
},
|
|
{
|
|
"epoch": 0.3387243862946902,
|
|
"grad_norm": 8.283249004397994,
|
|
"learning_rate": 4.1811300750410137e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4178,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"epoch": 0.33909661089501403,
|
|
"grad_norm": 6.208206273735844,
|
|
"learning_rate": 4.1787243834510793e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4196,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 9110
|
|
},
|
|
{
|
|
"epoch": 0.3394688354953379,
|
|
"grad_norm": 6.669403185375734,
|
|
"learning_rate": 4.176315857609963e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4304,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"epoch": 0.33984106009566173,
|
|
"grad_norm": 6.434839914194607,
|
|
"learning_rate": 4.173904501584066e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4271,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9130
|
|
},
|
|
{
|
|
"epoch": 0.34021328469598555,
|
|
"grad_norm": 6.27853766790502,
|
|
"learning_rate": 4.1714903194445686e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4217,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"epoch": 0.3405855092963094,
|
|
"grad_norm": 9.105893065582725,
|
|
"learning_rate": 4.169073315267421e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4118,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9150
|
|
},
|
|
{
|
|
"epoch": 0.34095773389663325,
|
|
"grad_norm": 6.875435651786618,
|
|
"learning_rate": 4.1666534931333406e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4209,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"epoch": 0.3413299584969571,
|
|
"grad_norm": 6.786260650352042,
|
|
"learning_rate": 4.1642308571277996e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4189,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9170
|
|
},
|
|
{
|
|
"epoch": 0.3417021830972809,
|
|
"grad_norm": 5.698368402847716,
|
|
"learning_rate": 4.1618054113410217e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3933,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"epoch": 0.3420744076976047,
|
|
"grad_norm": 6.952095882586489,
|
|
"learning_rate": 4.159377159867976e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4129,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9190
|
|
},
|
|
{
|
|
"epoch": 0.34244663229792854,
|
|
"grad_norm": 5.850135960583494,
|
|
"learning_rate": 4.1569461068083664e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"epoch": 0.3428188568982524,
|
|
"grad_norm": 6.146542333464643,
|
|
"learning_rate": 4.154512256266629e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.4171,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 9210
|
|
},
|
|
{
|
|
"epoch": 0.34319108149857624,
|
|
"grad_norm": 5.97886901598888,
|
|
"learning_rate": 4.152075612351921e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.3955,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"epoch": 0.34356330609890007,
|
|
"grad_norm": 6.71364261876551,
|
|
"learning_rate": 4.149636179178117e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4243,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9230
|
|
},
|
|
{
|
|
"epoch": 0.3439355306992239,
|
|
"grad_norm": 6.884692592393682,
|
|
"learning_rate": 4.1471939608637997e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"epoch": 0.34430775529954777,
|
|
"grad_norm": 6.071302294765108,
|
|
"learning_rate": 4.144748961532255e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4061,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9250
|
|
},
|
|
{
|
|
"epoch": 0.3446799798998716,
|
|
"grad_norm": 6.698432883381243,
|
|
"learning_rate": 4.1423011853114644e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4203,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"epoch": 0.3450522045001954,
|
|
"grad_norm": 5.822996174621156,
|
|
"learning_rate": 4.1398506363340965e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4018,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 9270
|
|
},
|
|
{
|
|
"epoch": 0.34542442910051924,
|
|
"grad_norm": 6.298107470087667,
|
|
"learning_rate": 4.137397318737502e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"epoch": 0.3457966537008431,
|
|
"grad_norm": 7.357744810723383,
|
|
"learning_rate": 4.134941236663706e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4217,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9290
|
|
},
|
|
{
|
|
"epoch": 0.34616887830116694,
|
|
"grad_norm": 7.3558803264435095,
|
|
"learning_rate": 4.132482394259401e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.4099,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"epoch": 0.34654110290149076,
|
|
"grad_norm": 6.240883295217108,
|
|
"learning_rate": 4.1300207956759395e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4239,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9310
|
|
},
|
|
{
|
|
"epoch": 0.3469133275018146,
|
|
"grad_norm": 6.970894711514693,
|
|
"learning_rate": 4.127556445069328e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4069,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"epoch": 0.3472855521021384,
|
|
"grad_norm": 7.136146202820999,
|
|
"learning_rate": 4.125089346600218e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4092,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9330
|
|
},
|
|
{
|
|
"epoch": 0.3476577767024623,
|
|
"grad_norm": 7.032592923553457,
|
|
"learning_rate": 4.122619504433902e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4206,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"epoch": 0.3480300013027861,
|
|
"grad_norm": 5.578080483299949,
|
|
"learning_rate": 4.120146922740303e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4159,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9350
|
|
},
|
|
{
|
|
"epoch": 0.34840222590310993,
|
|
"grad_norm": 5.528673313535459,
|
|
"learning_rate": 4.1176716056939715e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"epoch": 0.34877445050343375,
|
|
"grad_norm": 6.46712489092406,
|
|
"learning_rate": 4.115193557474074e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4092,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9370
|
|
},
|
|
{
|
|
"epoch": 0.34914667510375763,
|
|
"grad_norm": 6.86728024748694,
|
|
"learning_rate": 4.1127127822643894e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4133,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"epoch": 0.34951889970408145,
|
|
"grad_norm": 6.6586310848932,
|
|
"learning_rate": 4.1102292842533004e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4126,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9390
|
|
},
|
|
{
|
|
"epoch": 0.3498911243044053,
|
|
"grad_norm": 4.9953986320876655,
|
|
"learning_rate": 4.1077430676337867e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.405,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"epoch": 0.3502633489047291,
|
|
"grad_norm": 7.19618775434978,
|
|
"learning_rate": 4.1052541366034174e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.4018,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 9410
|
|
},
|
|
{
|
|
"epoch": 0.3506355735050529,
|
|
"grad_norm": 7.731108481797667,
|
|
"learning_rate": 4.102762495364346e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4289,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"epoch": 0.3510077981053768,
|
|
"grad_norm": 5.849057935096351,
|
|
"learning_rate": 4.100268148123299e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9430
|
|
},
|
|
{
|
|
"epoch": 0.3513800227057006,
|
|
"grad_norm": 5.54189743324542,
|
|
"learning_rate": 4.0977710990915747e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"epoch": 0.35175224730602445,
|
|
"grad_norm": 5.862439017008876,
|
|
"learning_rate": 4.0952713524850313e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4107,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9450
|
|
},
|
|
{
|
|
"epoch": 0.35212447190634827,
|
|
"grad_norm": 6.7294433609781255,
|
|
"learning_rate": 4.0927689125240806e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4275,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"epoch": 0.35249669650667215,
|
|
"grad_norm": 5.542091200007172,
|
|
"learning_rate": 4.090263783433683e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.435,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9470
|
|
},
|
|
{
|
|
"epoch": 0.35286892110699597,
|
|
"grad_norm": 6.639174527773383,
|
|
"learning_rate": 4.0877559694433384e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"epoch": 0.3532411457073198,
|
|
"grad_norm": 7.359094817030376,
|
|
"learning_rate": 4.0852454747870807e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4344,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9490
|
|
},
|
|
{
|
|
"epoch": 0.3536133703076436,
|
|
"grad_norm": 6.545935248321405,
|
|
"learning_rate": 4.082732303703469e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -820.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.4126,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"epoch": 0.3539855949079675,
|
|
"grad_norm": 5.968054422175286,
|
|
"learning_rate": 4.0802164604355805e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4083,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9510
|
|
},
|
|
{
|
|
"epoch": 0.3543578195082913,
|
|
"grad_norm": 6.526012259525071,
|
|
"learning_rate": 4.077697949231005e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4163,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"epoch": 0.35473004410861514,
|
|
"grad_norm": 6.880366350953438,
|
|
"learning_rate": 4.075176774341835e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.4116,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 9530
|
|
},
|
|
{
|
|
"epoch": 0.35510226870893896,
|
|
"grad_norm": 6.483557917102565,
|
|
"learning_rate": 4.0726529400246634e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.4139,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"epoch": 0.3554744933092628,
|
|
"grad_norm": 6.541046197839035,
|
|
"learning_rate": 4.070126450540569e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.411,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9550
|
|
},
|
|
{
|
|
"epoch": 0.35584671790958666,
|
|
"grad_norm": 5.704995921378098,
|
|
"learning_rate": 4.0675973101551177e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4026,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"epoch": 0.3562189425099105,
|
|
"grad_norm": 6.6939980321479915,
|
|
"learning_rate": 4.065065523138347e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4145,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9570
|
|
},
|
|
{
|
|
"epoch": 0.3565911671102343,
|
|
"grad_norm": 10.286463291727507,
|
|
"learning_rate": 4.062531093764764e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4133,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"epoch": 0.35696339171055813,
|
|
"grad_norm": 6.240640847775624,
|
|
"learning_rate": 4.05999402631334e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.413,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9590
|
|
},
|
|
{
|
|
"epoch": 0.357335616310882,
|
|
"grad_norm": 5.7953061492719895,
|
|
"learning_rate": 4.0574543250674973e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4207,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"epoch": 0.35770784091120583,
|
|
"grad_norm": 6.151665591033918,
|
|
"learning_rate": 4.054911994315104e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3904,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9610
|
|
},
|
|
{
|
|
"epoch": 0.35808006551152965,
|
|
"grad_norm": 7.724056079487011,
|
|
"learning_rate": 4.052367038348471e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.3775,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"epoch": 0.3584522901118535,
|
|
"grad_norm": 6.531161377753698,
|
|
"learning_rate": 4.049819461464338e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9630
|
|
},
|
|
{
|
|
"epoch": 0.3588245147121773,
|
|
"grad_norm": 7.062915441804796,
|
|
"learning_rate": 4.0472692679638733e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3979,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"epoch": 0.3591967393125012,
|
|
"grad_norm": 10.243709215827634,
|
|
"learning_rate": 4.0447164621526586e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3967,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 9650
|
|
},
|
|
{
|
|
"epoch": 0.359568963912825,
|
|
"grad_norm": 6.09437279859335,
|
|
"learning_rate": 4.04216104834069e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4109,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"epoch": 0.3599411885131488,
|
|
"grad_norm": 7.0834473527037165,
|
|
"learning_rate": 4.0396030308423643e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4302,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 9670
|
|
},
|
|
{
|
|
"epoch": 0.36031341311347265,
|
|
"grad_norm": 6.095238905555106,
|
|
"learning_rate": 4.037042413976476e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"epoch": 0.3606856377137965,
|
|
"grad_norm": 5.5030409006530725,
|
|
"learning_rate": 4.0344792020662067e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3855,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9690
|
|
},
|
|
{
|
|
"epoch": 0.36105786231412035,
|
|
"grad_norm": 7.250256115255806,
|
|
"learning_rate": 4.0319133994391206e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4189,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"epoch": 0.36143008691444417,
|
|
"grad_norm": 6.441598010604168,
|
|
"learning_rate": 4.0293450104271544e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4102,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9710
|
|
},
|
|
{
|
|
"epoch": 0.361802311514768,
|
|
"grad_norm": 6.773528920002974,
|
|
"learning_rate": 4.026774039366612e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.404,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"epoch": 0.36217453611509187,
|
|
"grad_norm": 6.876868653444924,
|
|
"learning_rate": 4.0242004905981587e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4301,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9730
|
|
},
|
|
{
|
|
"epoch": 0.3625467607154157,
|
|
"grad_norm": 6.354287464530844,
|
|
"learning_rate": 4.0216243684668073e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"epoch": 0.3629189853157395,
|
|
"grad_norm": 8.013897971181141,
|
|
"learning_rate": 4.019045677321921e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9750
|
|
},
|
|
{
|
|
"epoch": 0.36329120991606334,
|
|
"grad_norm": 5.972597439875224,
|
|
"learning_rate": 4.016464421517196e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4049,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"epoch": 0.36366343451638716,
|
|
"grad_norm": 5.760478609996771,
|
|
"learning_rate": 4.0138806054106604e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9770
|
|
},
|
|
{
|
|
"epoch": 0.36403565911671104,
|
|
"grad_norm": 6.8187297576482075,
|
|
"learning_rate": 4.011294233364664e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3928,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"epoch": 0.36440788371703486,
|
|
"grad_norm": 6.077472359176952,
|
|
"learning_rate": 4.0087053097458735e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.375,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 9790
|
|
},
|
|
{
|
|
"epoch": 0.3647801083173587,
|
|
"grad_norm": 7.558862444911248,
|
|
"learning_rate": 4.00611383892526e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4482,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"epoch": 0.3651523329176825,
|
|
"grad_norm": 6.7384897044264145,
|
|
"learning_rate": 4.003519825278101e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4252,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9810
|
|
},
|
|
{
|
|
"epoch": 0.3655245575180064,
|
|
"grad_norm": 6.987283324033663,
|
|
"learning_rate": 4.000923273183961e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"epoch": 0.3658967821183302,
|
|
"grad_norm": 6.05933441587129,
|
|
"learning_rate": 3.9983241870266935e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3919,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9830
|
|
},
|
|
{
|
|
"epoch": 0.36626900671865403,
|
|
"grad_norm": 6.529010763233826,
|
|
"learning_rate": 3.995722571194431e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4073,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"epoch": 0.36664123131897786,
|
|
"grad_norm": 7.803463998982988,
|
|
"learning_rate": 3.9931184300795746e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4128,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9850
|
|
},
|
|
{
|
|
"epoch": 0.36701345591930173,
|
|
"grad_norm": 6.249855727102126,
|
|
"learning_rate": 3.9905117680787906e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4231,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"epoch": 0.36738568051962556,
|
|
"grad_norm": 5.671708959671821,
|
|
"learning_rate": 3.987902589593e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4168,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9870
|
|
},
|
|
{
|
|
"epoch": 0.3677579051199494,
|
|
"grad_norm": 5.452023733195032,
|
|
"learning_rate": 3.9852908990273737e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4047,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"epoch": 0.3681301297202732,
|
|
"grad_norm": 5.643034118918705,
|
|
"learning_rate": 3.9826767007913246e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 9890
|
|
},
|
|
{
|
|
"epoch": 0.368502354320597,
|
|
"grad_norm": 5.949504261642717,
|
|
"learning_rate": 3.9800599992984973e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4144,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"epoch": 0.3688745789209209,
|
|
"grad_norm": 6.7321939179405526,
|
|
"learning_rate": 3.9774407989667637e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9910
|
|
},
|
|
{
|
|
"epoch": 0.3692468035212447,
|
|
"grad_norm": 6.042659898746077,
|
|
"learning_rate": 3.9748191042182143e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"epoch": 0.36961902812156855,
|
|
"grad_norm": 6.836182453809051,
|
|
"learning_rate": 3.9721949194791527e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3979,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 9930
|
|
},
|
|
{
|
|
"epoch": 0.36999125272189237,
|
|
"grad_norm": 5.827364938139911,
|
|
"learning_rate": 3.969568249180083e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4351,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"epoch": 0.37036347732221625,
|
|
"grad_norm": 5.892075689843871,
|
|
"learning_rate": 3.96693909775571e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 9950
|
|
},
|
|
{
|
|
"epoch": 0.3707357019225401,
|
|
"grad_norm": 6.905053086738602,
|
|
"learning_rate": 3.9643074696449235e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4052,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"epoch": 0.3711079265228639,
|
|
"grad_norm": 6.39690664427754,
|
|
"learning_rate": 3.961673369290798e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4034,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 9970
|
|
},
|
|
{
|
|
"epoch": 0.3714801511231877,
|
|
"grad_norm": 6.412933081381276,
|
|
"learning_rate": 3.9590368011405786e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.3882,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"epoch": 0.37185237572351154,
|
|
"grad_norm": 6.635149179644065,
|
|
"learning_rate": 3.956397769645681e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.3986,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 9990
|
|
},
|
|
{
|
|
"epoch": 0.3722246003238354,
|
|
"grad_norm": 7.024506799311308,
|
|
"learning_rate": 3.9537562792616753e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.413,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.3722246003238354,
|
|
"eval_logits/chosen": -3.890625,
|
|
"eval_logits/rejected": -3.71875,
|
|
"eval_logps/chosen": -912.0,
|
|
"eval_logps/rejected": -1020.0,
|
|
"eval_loss": 0.4581940472126007,
|
|
"eval_rewards/accuracies": 0.7553164958953857,
|
|
"eval_rewards/chosen": -7.09375,
|
|
"eval_rewards/margins": 1.40625,
|
|
"eval_rewards/rejected": -8.5,
|
|
"eval_runtime": 6317.7485,
|
|
"eval_samples_per_second": 30.241,
|
|
"eval_steps_per_second": 0.473,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.37259682492415924,
|
|
"grad_norm": 6.620756048562296,
|
|
"learning_rate": 3.951112334448288e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10010
|
|
},
|
|
{
|
|
"epoch": 0.37296904952448307,
|
|
"grad_norm": 7.514335970027951,
|
|
"learning_rate": 3.948465939669385e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4081,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 10020
|
|
},
|
|
{
|
|
"epoch": 0.3733412741248069,
|
|
"grad_norm": 6.805966697385053,
|
|
"learning_rate": 3.9458170993929705e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4463,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 10030
|
|
},
|
|
{
|
|
"epoch": 0.37371349872513077,
|
|
"grad_norm": 5.373682341279974,
|
|
"learning_rate": 3.94316581809118e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.4141,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 10040
|
|
},
|
|
{
|
|
"epoch": 0.3740857233254546,
|
|
"grad_norm": 5.9826136703232295,
|
|
"learning_rate": 3.940512100240264e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -4.03125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 10050
|
|
},
|
|
{
|
|
"epoch": 0.3744579479257784,
|
|
"grad_norm": 6.409181109952671,
|
|
"learning_rate": 3.9378559503205934e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -828.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10060
|
|
},
|
|
{
|
|
"epoch": 0.37483017252610223,
|
|
"grad_norm": 7.102541698272061,
|
|
"learning_rate": 3.9351973728166407e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -816.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4092,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10070
|
|
},
|
|
{
|
|
"epoch": 0.3752023971264261,
|
|
"grad_norm": 5.309064927319129,
|
|
"learning_rate": 3.9325363722169787e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10080
|
|
},
|
|
{
|
|
"epoch": 0.37557462172674994,
|
|
"grad_norm": 5.482064357214902,
|
|
"learning_rate": 3.9298729530142716e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 10090
|
|
},
|
|
{
|
|
"epoch": 0.37594684632707376,
|
|
"grad_norm": 7.606405920380892,
|
|
"learning_rate": 3.927207119705267e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4178,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10100
|
|
},
|
|
{
|
|
"epoch": 0.3763190709273976,
|
|
"grad_norm": 6.564568634427273,
|
|
"learning_rate": 3.924538876790787e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3836,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10110
|
|
},
|
|
{
|
|
"epoch": 0.3766912955277214,
|
|
"grad_norm": 7.088816984475297,
|
|
"learning_rate": 3.9218682287757233e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4089,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10120
|
|
},
|
|
{
|
|
"epoch": 0.3770635201280453,
|
|
"grad_norm": 7.313108564826078,
|
|
"learning_rate": 3.9191951801690273e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 10130
|
|
},
|
|
{
|
|
"epoch": 0.3774357447283691,
|
|
"grad_norm": 6.471114065647866,
|
|
"learning_rate": 3.916519735483703e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -832.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.404,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10140
|
|
},
|
|
{
|
|
"epoch": 0.3778079693286929,
|
|
"grad_norm": 7.010437395229217,
|
|
"learning_rate": 3.913841899236803e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10150
|
|
},
|
|
{
|
|
"epoch": 0.37818019392901675,
|
|
"grad_norm": 6.024243908667659,
|
|
"learning_rate": 3.911161675949412e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -836.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4062,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10160
|
|
},
|
|
{
|
|
"epoch": 0.37855241852934063,
|
|
"grad_norm": 6.416717621903686,
|
|
"learning_rate": 3.90847907014665e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4136,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10170
|
|
},
|
|
{
|
|
"epoch": 0.37892464312966445,
|
|
"grad_norm": 6.851128018572843,
|
|
"learning_rate": 3.905794086357658e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4186,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10180
|
|
},
|
|
{
|
|
"epoch": 0.3792968677299883,
|
|
"grad_norm": 6.72728296805784,
|
|
"learning_rate": 3.903106729115591e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4112,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10190
|
|
},
|
|
{
|
|
"epoch": 0.3796690923303121,
|
|
"grad_norm": 6.954862351394934,
|
|
"learning_rate": 3.90041700295761e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.403,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"epoch": 0.3800413169306359,
|
|
"grad_norm": 5.8342721921191325,
|
|
"learning_rate": 3.897724912424879e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 10210
|
|
},
|
|
{
|
|
"epoch": 0.3804135415309598,
|
|
"grad_norm": 5.1804967155317545,
|
|
"learning_rate": 3.8950304620625514e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4133,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10220
|
|
},
|
|
{
|
|
"epoch": 0.3807857661312836,
|
|
"grad_norm": 7.073584493897113,
|
|
"learning_rate": 3.892333656419765e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4279,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10230
|
|
},
|
|
{
|
|
"epoch": 0.38115799073160744,
|
|
"grad_norm": 5.477118219815857,
|
|
"learning_rate": 3.8896345000496343e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10240
|
|
},
|
|
{
|
|
"epoch": 0.38153021533193127,
|
|
"grad_norm": 5.7793543009252195,
|
|
"learning_rate": 3.886932997509244e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3717,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10250
|
|
},
|
|
{
|
|
"epoch": 0.38190243993225514,
|
|
"grad_norm": 9.11464193426846,
|
|
"learning_rate": 3.884229153359637e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4225,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 10260
|
|
},
|
|
{
|
|
"epoch": 0.38227466453257897,
|
|
"grad_norm": 5.65179391628191,
|
|
"learning_rate": 3.881522972165812e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10270
|
|
},
|
|
{
|
|
"epoch": 0.3826468891329028,
|
|
"grad_norm": 6.436096076025931,
|
|
"learning_rate": 3.8788144584967135e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3847,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10280
|
|
},
|
|
{
|
|
"epoch": 0.3830191137332266,
|
|
"grad_norm": 7.6972390699351765,
|
|
"learning_rate": 3.876103616925223e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4059,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10290
|
|
},
|
|
{
|
|
"epoch": 0.3833913383335505,
|
|
"grad_norm": 7.114339365106922,
|
|
"learning_rate": 3.873390452028151e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3919,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 10300
|
|
},
|
|
{
|
|
"epoch": 0.3837635629338743,
|
|
"grad_norm": 6.002128507146674,
|
|
"learning_rate": 3.870674968386234e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 10310
|
|
},
|
|
{
|
|
"epoch": 0.38413578753419814,
|
|
"grad_norm": 6.60128909786954,
|
|
"learning_rate": 3.86795717058412e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3972,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10320
|
|
},
|
|
{
|
|
"epoch": 0.38450801213452196,
|
|
"grad_norm": 7.388287282400049,
|
|
"learning_rate": 3.8652370632103665e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 10330
|
|
},
|
|
{
|
|
"epoch": 0.3848802367348458,
|
|
"grad_norm": 6.322555250358002,
|
|
"learning_rate": 3.862514650857428e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4367,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10340
|
|
},
|
|
{
|
|
"epoch": 0.38525246133516966,
|
|
"grad_norm": 5.63165919875538,
|
|
"learning_rate": 3.859789938121654e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4124,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10350
|
|
},
|
|
{
|
|
"epoch": 0.3856246859354935,
|
|
"grad_norm": 7.237845230428056,
|
|
"learning_rate": 3.8570629296032734e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 10360
|
|
},
|
|
{
|
|
"epoch": 0.3859969105358173,
|
|
"grad_norm": 6.743152400415412,
|
|
"learning_rate": 3.854333629906395e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4335,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 10370
|
|
},
|
|
{
|
|
"epoch": 0.38636913513614113,
|
|
"grad_norm": 6.636759347005598,
|
|
"learning_rate": 3.851602043638994e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3988,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10380
|
|
},
|
|
{
|
|
"epoch": 0.386741359736465,
|
|
"grad_norm": 5.786548322572369,
|
|
"learning_rate": 3.848868175412906e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10390
|
|
},
|
|
{
|
|
"epoch": 0.38711358433678883,
|
|
"grad_norm": 5.931567599592094,
|
|
"learning_rate": 3.8461320298438206e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.417,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.2890625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"epoch": 0.38748580893711265,
|
|
"grad_norm": 6.177815324462008,
|
|
"learning_rate": 3.84339361155127e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4045,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10410
|
|
},
|
|
{
|
|
"epoch": 0.3878580335374365,
|
|
"grad_norm": 7.240409108639424,
|
|
"learning_rate": 3.8406529251586254e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4091,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 10420
|
|
},
|
|
{
|
|
"epoch": 0.3882302581377603,
|
|
"grad_norm": 7.427897210395563,
|
|
"learning_rate": 3.837909975293088e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4075,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10430
|
|
},
|
|
{
|
|
"epoch": 0.3886024827380842,
|
|
"grad_norm": 7.540006649363661,
|
|
"learning_rate": 3.835164766585679e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10440
|
|
},
|
|
{
|
|
"epoch": 0.388974707338408,
|
|
"grad_norm": 6.767131861155957,
|
|
"learning_rate": 3.8324173036712336e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10450
|
|
},
|
|
{
|
|
"epoch": 0.3893469319387318,
|
|
"grad_norm": 6.547527954837288,
|
|
"learning_rate": 3.829667591188393e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10460
|
|
},
|
|
{
|
|
"epoch": 0.38971915653905564,
|
|
"grad_norm": 5.74922708779227,
|
|
"learning_rate": 3.826915633779596e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10470
|
|
},
|
|
{
|
|
"epoch": 0.3900913811393795,
|
|
"grad_norm": 7.463387676001147,
|
|
"learning_rate": 3.8241614360910726e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10480
|
|
},
|
|
{
|
|
"epoch": 0.39046360573970335,
|
|
"grad_norm": 11.960834677593498,
|
|
"learning_rate": 3.8214050027728335e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10490
|
|
},
|
|
{
|
|
"epoch": 0.39083583034002717,
|
|
"grad_norm": 6.100049806059151,
|
|
"learning_rate": 3.818646338478665e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.418,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"epoch": 0.391208054940351,
|
|
"grad_norm": 6.395784708644026,
|
|
"learning_rate": 3.815885447866121e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4388,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10510
|
|
},
|
|
{
|
|
"epoch": 0.39158027954067487,
|
|
"grad_norm": 6.607906649659632,
|
|
"learning_rate": 3.813122335596513e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4264,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10520
|
|
},
|
|
{
|
|
"epoch": 0.3919525041409987,
|
|
"grad_norm": 6.543038322919457,
|
|
"learning_rate": 3.8103570063349027e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10530
|
|
},
|
|
{
|
|
"epoch": 0.3923247287413225,
|
|
"grad_norm": 5.938039519137844,
|
|
"learning_rate": 3.807589464750097e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.3984,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10540
|
|
},
|
|
{
|
|
"epoch": 0.39269695334164634,
|
|
"grad_norm": 6.812210012906291,
|
|
"learning_rate": 3.8048197155146343e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3901,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10550
|
|
},
|
|
{
|
|
"epoch": 0.39306917794197016,
|
|
"grad_norm": 6.154719047402081,
|
|
"learning_rate": 3.8020477633047847e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4349,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 10560
|
|
},
|
|
{
|
|
"epoch": 0.39344140254229404,
|
|
"grad_norm": 6.719196748437144,
|
|
"learning_rate": 3.7992736128005344e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4482,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10570
|
|
},
|
|
{
|
|
"epoch": 0.39381362714261786,
|
|
"grad_norm": 6.448700351661077,
|
|
"learning_rate": 3.7964972686855833e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10580
|
|
},
|
|
{
|
|
"epoch": 0.3941858517429417,
|
|
"grad_norm": 7.277129455696457,
|
|
"learning_rate": 3.7937187356473323e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3942,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10590
|
|
},
|
|
{
|
|
"epoch": 0.3945580763432655,
|
|
"grad_norm": 6.626728978935036,
|
|
"learning_rate": 3.7909380183768796e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"epoch": 0.3949303009435894,
|
|
"grad_norm": 6.771630164839315,
|
|
"learning_rate": 3.7881551215690123e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4285,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 10610
|
|
},
|
|
{
|
|
"epoch": 0.3953025255439132,
|
|
"grad_norm": 7.075349186396898,
|
|
"learning_rate": 3.785370049922194e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10620
|
|
},
|
|
{
|
|
"epoch": 0.39567475014423703,
|
|
"grad_norm": 5.967375555574234,
|
|
"learning_rate": 3.782582808138564e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4162,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 10630
|
|
},
|
|
{
|
|
"epoch": 0.39604697474456085,
|
|
"grad_norm": 6.486449411354829,
|
|
"learning_rate": 3.7797934009239217e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10640
|
|
},
|
|
{
|
|
"epoch": 0.3964191993448847,
|
|
"grad_norm": 7.453505278617288,
|
|
"learning_rate": 3.777001832987726e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4276,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10650
|
|
},
|
|
{
|
|
"epoch": 0.39679142394520855,
|
|
"grad_norm": 6.689005076459521,
|
|
"learning_rate": 3.77420810904308e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4138,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10660
|
|
},
|
|
{
|
|
"epoch": 0.3971636485455324,
|
|
"grad_norm": 7.1005147287850106,
|
|
"learning_rate": 3.771412233806731e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4023,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10670
|
|
},
|
|
{
|
|
"epoch": 0.3975358731458562,
|
|
"grad_norm": 6.367244389070103,
|
|
"learning_rate": 3.768614211999056e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10680
|
|
},
|
|
{
|
|
"epoch": 0.39790809774618,
|
|
"grad_norm": 6.362748587039061,
|
|
"learning_rate": 3.765814048344056e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.41,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10690
|
|
},
|
|
{
|
|
"epoch": 0.3982803223465039,
|
|
"grad_norm": 6.954124107165623,
|
|
"learning_rate": 3.76301174756935e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4037,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10700
|
|
},
|
|
{
|
|
"epoch": 0.3986525469468277,
|
|
"grad_norm": 6.082109877189924,
|
|
"learning_rate": 3.7602073144061617e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.3961,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10710
|
|
},
|
|
{
|
|
"epoch": 0.39902477154715155,
|
|
"grad_norm": 6.675705674846204,
|
|
"learning_rate": 3.757400753589319e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 10720
|
|
},
|
|
{
|
|
"epoch": 0.39939699614747537,
|
|
"grad_norm": 6.4566289660499585,
|
|
"learning_rate": 3.7545920698572393e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 10730
|
|
},
|
|
{
|
|
"epoch": 0.39976922074779925,
|
|
"grad_norm": 6.990911078434632,
|
|
"learning_rate": 3.7517812679519255e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4118,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 10740
|
|
},
|
|
{
|
|
"epoch": 0.40014144534812307,
|
|
"grad_norm": 6.415320216470397,
|
|
"learning_rate": 3.748968352618957e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10750
|
|
},
|
|
{
|
|
"epoch": 0.4005136699484469,
|
|
"grad_norm": 6.497170635847852,
|
|
"learning_rate": 3.7461533286074785e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3974,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10760
|
|
},
|
|
{
|
|
"epoch": 0.4008858945487707,
|
|
"grad_norm": 7.6654676689450785,
|
|
"learning_rate": 3.743336200670199e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3766,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10770
|
|
},
|
|
{
|
|
"epoch": 0.40125811914909454,
|
|
"grad_norm": 5.901867150463746,
|
|
"learning_rate": 3.7405169735633766e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4132,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10780
|
|
},
|
|
{
|
|
"epoch": 0.4016303437494184,
|
|
"grad_norm": 6.7076566098295505,
|
|
"learning_rate": 3.7376956520468154e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10790
|
|
},
|
|
{
|
|
"epoch": 0.40200256834974224,
|
|
"grad_norm": 6.386232880541396,
|
|
"learning_rate": 3.734872240883854e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.409,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"epoch": 0.40237479295006606,
|
|
"grad_norm": 6.640224929469324,
|
|
"learning_rate": 3.73204674484136e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.71875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10810
|
|
},
|
|
{
|
|
"epoch": 0.4027470175503899,
|
|
"grad_norm": 6.795427493111171,
|
|
"learning_rate": 3.729219168689721e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -848.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.4114,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.53125,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 10820
|
|
},
|
|
{
|
|
"epoch": 0.40311924215071376,
|
|
"grad_norm": 6.533267351367528,
|
|
"learning_rate": 3.7263895172028345e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4029,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10830
|
|
},
|
|
{
|
|
"epoch": 0.4034914667510376,
|
|
"grad_norm": 7.743230530498027,
|
|
"learning_rate": 3.7235577951581063e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.4256,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10840
|
|
},
|
|
{
|
|
"epoch": 0.4038636913513614,
|
|
"grad_norm": 6.460380533771181,
|
|
"learning_rate": 3.7207240073364334e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4132,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10850
|
|
},
|
|
{
|
|
"epoch": 0.40423591595168523,
|
|
"grad_norm": 7.712329231730129,
|
|
"learning_rate": 3.7178881585222035e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4004,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.375,
|
|
"step": 10860
|
|
},
|
|
{
|
|
"epoch": 0.40460814055200905,
|
|
"grad_norm": 7.150228565668666,
|
|
"learning_rate": 3.7150502535032825e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4228,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 10870
|
|
},
|
|
{
|
|
"epoch": 0.40498036515233293,
|
|
"grad_norm": 6.4128192869498415,
|
|
"learning_rate": 3.7122102970710065e-07,
|
|
"logits/chosen": -4.125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4158,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 10880
|
|
},
|
|
{
|
|
"epoch": 0.40535258975265676,
|
|
"grad_norm": 6.3870531453301,
|
|
"learning_rate": 3.7093682940201803e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3979,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 10890
|
|
},
|
|
{
|
|
"epoch": 0.4057248143529806,
|
|
"grad_norm": 6.643823061203458,
|
|
"learning_rate": 3.7065242491490576e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4109,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 10900
|
|
},
|
|
{
|
|
"epoch": 0.4060970389533044,
|
|
"grad_norm": 7.222682190007728,
|
|
"learning_rate": 3.703678167259344e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4112,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10910
|
|
},
|
|
{
|
|
"epoch": 0.4064692635536283,
|
|
"grad_norm": 6.3472174489137965,
|
|
"learning_rate": 3.700830053156182e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4072,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 10920
|
|
},
|
|
{
|
|
"epoch": 0.4068414881539521,
|
|
"grad_norm": 7.07728767392971,
|
|
"learning_rate": 3.6979799116481457e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3766,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10930
|
|
},
|
|
{
|
|
"epoch": 0.4072137127542759,
|
|
"grad_norm": 5.646009883088342,
|
|
"learning_rate": 3.6951277475472333e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4017,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10940
|
|
},
|
|
{
|
|
"epoch": 0.40758593735459975,
|
|
"grad_norm": 8.369092605874348,
|
|
"learning_rate": 3.6922735656688545e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10950
|
|
},
|
|
{
|
|
"epoch": 0.4079581619549236,
|
|
"grad_norm": 5.900399445134576,
|
|
"learning_rate": 3.6894173708318305e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.65625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 10960
|
|
},
|
|
{
|
|
"epoch": 0.40833038655524745,
|
|
"grad_norm": 6.3201249297109126,
|
|
"learning_rate": 3.686559167858377e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 10970
|
|
},
|
|
{
|
|
"epoch": 0.40870261115557127,
|
|
"grad_norm": 5.965186280342673,
|
|
"learning_rate": 3.683698961574102e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4011,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 10980
|
|
},
|
|
{
|
|
"epoch": 0.4090748357558951,
|
|
"grad_norm": 6.651087266678326,
|
|
"learning_rate": 3.680836756807995e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4166,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 10990
|
|
},
|
|
{
|
|
"epoch": 0.4094470603562189,
|
|
"grad_norm": 7.54335850268088,
|
|
"learning_rate": 3.677972558392421e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3928,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11000
|
|
},
|
|
{
|
|
"epoch": 0.4098192849565428,
|
|
"grad_norm": 6.489031085016841,
|
|
"learning_rate": 3.675106371163109e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4314,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11010
|
|
},
|
|
{
|
|
"epoch": 0.4101915095568666,
|
|
"grad_norm": 6.1330111383165455,
|
|
"learning_rate": 3.672238199959146e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 11020
|
|
},
|
|
{
|
|
"epoch": 0.41056373415719044,
|
|
"grad_norm": 6.247381282677274,
|
|
"learning_rate": 3.6693680496229717e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4151,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11030
|
|
},
|
|
{
|
|
"epoch": 0.41093595875751426,
|
|
"grad_norm": 6.561128596232543,
|
|
"learning_rate": 3.6664959250003623e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4087,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11040
|
|
},
|
|
{
|
|
"epoch": 0.41130818335783814,
|
|
"grad_norm": 6.293852729040782,
|
|
"learning_rate": 3.663621830940432e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4265,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11050
|
|
},
|
|
{
|
|
"epoch": 0.41168040795816196,
|
|
"grad_norm": 6.358075787766797,
|
|
"learning_rate": 3.6607457722956163e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4116,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.2265625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11060
|
|
},
|
|
{
|
|
"epoch": 0.4120526325584858,
|
|
"grad_norm": 6.8711539718448815,
|
|
"learning_rate": 3.6578677539216704e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4105,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11070
|
|
},
|
|
{
|
|
"epoch": 0.4124248571588096,
|
|
"grad_norm": 8.474085776166392,
|
|
"learning_rate": 3.6549877806776555e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4148,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11080
|
|
},
|
|
{
|
|
"epoch": 0.4127970817591335,
|
|
"grad_norm": 7.055181814440708,
|
|
"learning_rate": 3.6521058574259355e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4344,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11090
|
|
},
|
|
{
|
|
"epoch": 0.4131693063594573,
|
|
"grad_norm": 6.413588750748051,
|
|
"learning_rate": 3.6492219890321674e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.4273,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 11100
|
|
},
|
|
{
|
|
"epoch": 0.41354153095978113,
|
|
"grad_norm": 5.671816849155975,
|
|
"learning_rate": 3.6463361803652877e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3827,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 11110
|
|
},
|
|
{
|
|
"epoch": 0.41391375556010496,
|
|
"grad_norm": 7.153131721739585,
|
|
"learning_rate": 3.6434484362975135e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11120
|
|
},
|
|
{
|
|
"epoch": 0.4142859801604288,
|
|
"grad_norm": 7.672517406155899,
|
|
"learning_rate": 3.640558761704328e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4212,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11130
|
|
},
|
|
{
|
|
"epoch": 0.41465820476075266,
|
|
"grad_norm": 7.826665109009803,
|
|
"learning_rate": 3.6376671614644726e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -856.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4084,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 11140
|
|
},
|
|
{
|
|
"epoch": 0.4150304293610765,
|
|
"grad_norm": 6.634217759035235,
|
|
"learning_rate": 3.6347736404599404e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4254,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11150
|
|
},
|
|
{
|
|
"epoch": 0.4154026539614003,
|
|
"grad_norm": 7.702040282829974,
|
|
"learning_rate": 3.631878203575969e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.384,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11160
|
|
},
|
|
{
|
|
"epoch": 0.4157748785617241,
|
|
"grad_norm": 6.464179475763968,
|
|
"learning_rate": 3.628980855701028e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3936,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11170
|
|
},
|
|
{
|
|
"epoch": 0.416147103162048,
|
|
"grad_norm": 5.489955044722155,
|
|
"learning_rate": 3.626081601726816e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11180
|
|
},
|
|
{
|
|
"epoch": 0.4165193277623718,
|
|
"grad_norm": 6.984581874057404,
|
|
"learning_rate": 3.623180446548248e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4083,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 11190
|
|
},
|
|
{
|
|
"epoch": 0.41689155236269565,
|
|
"grad_norm": 6.48136638380753,
|
|
"learning_rate": 3.620277395063449e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -844.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3836,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.6875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11200
|
|
},
|
|
{
|
|
"epoch": 0.4172637769630195,
|
|
"grad_norm": 6.456915698827229,
|
|
"learning_rate": 3.6173724521737467e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3911,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11210
|
|
},
|
|
{
|
|
"epoch": 0.4176360015633433,
|
|
"grad_norm": 6.221032503279322,
|
|
"learning_rate": 3.61446562278366e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11220
|
|
},
|
|
{
|
|
"epoch": 0.4180082261636672,
|
|
"grad_norm": 5.732175184316063,
|
|
"learning_rate": 3.6115569118008966e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11230
|
|
},
|
|
{
|
|
"epoch": 0.418380450763991,
|
|
"grad_norm": 6.643651378378301,
|
|
"learning_rate": 3.6086463241363365e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11240
|
|
},
|
|
{
|
|
"epoch": 0.4187526753643148,
|
|
"grad_norm": 6.368375451463488,
|
|
"learning_rate": 3.605733864704031e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4191,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 11250
|
|
},
|
|
{
|
|
"epoch": 0.41912489996463864,
|
|
"grad_norm": 6.417941113532811,
|
|
"learning_rate": 3.602819538421191e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4019,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11260
|
|
},
|
|
{
|
|
"epoch": 0.4194971245649625,
|
|
"grad_norm": 6.504630992835788,
|
|
"learning_rate": 3.5999033502081783e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4166,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 11270
|
|
},
|
|
{
|
|
"epoch": 0.41986934916528634,
|
|
"grad_norm": 6.0059143285438585,
|
|
"learning_rate": 3.596985304988501e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11280
|
|
},
|
|
{
|
|
"epoch": 0.42024157376561017,
|
|
"grad_norm": 7.316409566911938,
|
|
"learning_rate": 3.5940654076887977e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4218,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11290
|
|
},
|
|
{
|
|
"epoch": 0.420613798365934,
|
|
"grad_norm": 6.981909031281695,
|
|
"learning_rate": 3.5911436632388396e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4011,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11300
|
|
},
|
|
{
|
|
"epoch": 0.42098602296625787,
|
|
"grad_norm": 8.342752888682286,
|
|
"learning_rate": 3.588220076571513e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4202,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11310
|
|
},
|
|
{
|
|
"epoch": 0.4213582475665817,
|
|
"grad_norm": 6.266657503984341,
|
|
"learning_rate": 3.5852946526228136e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11320
|
|
},
|
|
{
|
|
"epoch": 0.4217304721669055,
|
|
"grad_norm": 7.198609948981566,
|
|
"learning_rate": 3.582367396331842e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.402,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11330
|
|
},
|
|
{
|
|
"epoch": 0.42210269676722934,
|
|
"grad_norm": 6.543719185010873,
|
|
"learning_rate": 3.5794383126407905e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11340
|
|
},
|
|
{
|
|
"epoch": 0.42247492136755316,
|
|
"grad_norm": 6.620101695268156,
|
|
"learning_rate": 3.5765074064949376e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3783,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 11350
|
|
},
|
|
{
|
|
"epoch": 0.42284714596787704,
|
|
"grad_norm": 8.445820489341617,
|
|
"learning_rate": 3.573574682842637e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 11360
|
|
},
|
|
{
|
|
"epoch": 0.42321937056820086,
|
|
"grad_norm": 6.762188411836831,
|
|
"learning_rate": 3.5706401466353146e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4179,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 11370
|
|
},
|
|
{
|
|
"epoch": 0.4235915951685247,
|
|
"grad_norm": 5.682049771246989,
|
|
"learning_rate": 3.567703802827453e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4129,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11380
|
|
},
|
|
{
|
|
"epoch": 0.4239638197688485,
|
|
"grad_norm": 7.002640802132963,
|
|
"learning_rate": 3.564765656376587e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11390
|
|
},
|
|
{
|
|
"epoch": 0.4243360443691724,
|
|
"grad_norm": 7.1802035752744136,
|
|
"learning_rate": 3.561825712243297e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11400
|
|
},
|
|
{
|
|
"epoch": 0.4247082689694962,
|
|
"grad_norm": 7.886403694405898,
|
|
"learning_rate": 3.5588839753911964e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3834,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11410
|
|
},
|
|
{
|
|
"epoch": 0.42508049356982003,
|
|
"grad_norm": 5.83236214029806,
|
|
"learning_rate": 3.555940450786926e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4162,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 11420
|
|
},
|
|
{
|
|
"epoch": 0.42545271817014385,
|
|
"grad_norm": 6.392227197130097,
|
|
"learning_rate": 3.5529951434001453e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11430
|
|
},
|
|
{
|
|
"epoch": 0.4258249427704677,
|
|
"grad_norm": 6.0699759198727,
|
|
"learning_rate": 3.5500480582035236e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11440
|
|
},
|
|
{
|
|
"epoch": 0.42619716737079155,
|
|
"grad_norm": 6.693110726299135,
|
|
"learning_rate": 3.547099200172731e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11450
|
|
},
|
|
{
|
|
"epoch": 0.4265693919711154,
|
|
"grad_norm": 17.88944429537948,
|
|
"learning_rate": 3.544148574286432e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11460
|
|
},
|
|
{
|
|
"epoch": 0.4269416165714392,
|
|
"grad_norm": 6.875583135067165,
|
|
"learning_rate": 3.541196185526275e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11470
|
|
},
|
|
{
|
|
"epoch": 0.427313841171763,
|
|
"grad_norm": 6.8358322396801245,
|
|
"learning_rate": 3.538242038876885e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3989,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 11480
|
|
},
|
|
{
|
|
"epoch": 0.4276860657720869,
|
|
"grad_norm": 6.0437843587197095,
|
|
"learning_rate": 3.5352861393258535e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11490
|
|
},
|
|
{
|
|
"epoch": 0.4280582903724107,
|
|
"grad_norm": 7.059850666105395,
|
|
"learning_rate": 3.5323284918637346e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3966,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11500
|
|
},
|
|
{
|
|
"epoch": 0.42843051497273454,
|
|
"grad_norm": 7.711862938147544,
|
|
"learning_rate": 3.529369101484031e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4338,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11510
|
|
},
|
|
{
|
|
"epoch": 0.42880273957305837,
|
|
"grad_norm": 6.60474789141993,
|
|
"learning_rate": 3.526407973183188e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3804,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11520
|
|
},
|
|
{
|
|
"epoch": 0.42917496417338225,
|
|
"grad_norm": 7.414049612055153,
|
|
"learning_rate": 3.5234451119605864e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 11530
|
|
},
|
|
{
|
|
"epoch": 0.42954718877370607,
|
|
"grad_norm": 6.047664236180073,
|
|
"learning_rate": 3.520480522818532e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4047,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 11540
|
|
},
|
|
{
|
|
"epoch": 0.4299194133740299,
|
|
"grad_norm": 6.425217979688092,
|
|
"learning_rate": 3.517514210762248e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.428,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11550
|
|
},
|
|
{
|
|
"epoch": 0.4302916379743537,
|
|
"grad_norm": 5.967288957981366,
|
|
"learning_rate": 3.514546180799867e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4206,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11560
|
|
},
|
|
{
|
|
"epoch": 0.43066386257467754,
|
|
"grad_norm": 6.102765663296686,
|
|
"learning_rate": 3.5115764379424204e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11570
|
|
},
|
|
{
|
|
"epoch": 0.4310360871750014,
|
|
"grad_norm": 6.134632632596522,
|
|
"learning_rate": 3.508604987203834e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4199,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11580
|
|
},
|
|
{
|
|
"epoch": 0.43140831177532524,
|
|
"grad_norm": 7.713928613463219,
|
|
"learning_rate": 3.505631833600914e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4062,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11590
|
|
},
|
|
{
|
|
"epoch": 0.43178053637564906,
|
|
"grad_norm": 6.74508762693865,
|
|
"learning_rate": 3.5026569821533445e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3647,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 11600
|
|
},
|
|
{
|
|
"epoch": 0.4321527609759729,
|
|
"grad_norm": 7.506027901591573,
|
|
"learning_rate": 3.4996804378836755e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3957,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 11610
|
|
},
|
|
{
|
|
"epoch": 0.43252498557629676,
|
|
"grad_norm": 7.856627169629348,
|
|
"learning_rate": 3.496702205817313e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.381,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11620
|
|
},
|
|
{
|
|
"epoch": 0.4328972101766206,
|
|
"grad_norm": 6.741367388751743,
|
|
"learning_rate": 3.4937222909825153e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4037,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 11630
|
|
},
|
|
{
|
|
"epoch": 0.4332694347769444,
|
|
"grad_norm": 6.970535681982311,
|
|
"learning_rate": 3.4907406984103794e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3938,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 11640
|
|
},
|
|
{
|
|
"epoch": 0.43364165937726823,
|
|
"grad_norm": 6.735200797914779,
|
|
"learning_rate": 3.4877574331348376e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3764,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 11650
|
|
},
|
|
{
|
|
"epoch": 0.43401388397759205,
|
|
"grad_norm": 6.501105066342323,
|
|
"learning_rate": 3.484772500192643e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4209,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11660
|
|
},
|
|
{
|
|
"epoch": 0.43438610857791593,
|
|
"grad_norm": 6.02726667188191,
|
|
"learning_rate": 3.481785904623368e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.4158,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.59375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 11670
|
|
},
|
|
{
|
|
"epoch": 0.43475833317823975,
|
|
"grad_norm": 6.398976098004674,
|
|
"learning_rate": 3.4787976514693873e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.4036,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -6.78125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 11680
|
|
},
|
|
{
|
|
"epoch": 0.4351305577785636,
|
|
"grad_norm": 6.779877738733095,
|
|
"learning_rate": 3.475807745775879e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4114,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11690
|
|
},
|
|
{
|
|
"epoch": 0.4355027823788874,
|
|
"grad_norm": 5.726779793726131,
|
|
"learning_rate": 3.4728161925908093e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.393,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11700
|
|
},
|
|
{
|
|
"epoch": 0.4358750069792113,
|
|
"grad_norm": 6.85617682777684,
|
|
"learning_rate": 3.4698229969649246e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 11710
|
|
},
|
|
{
|
|
"epoch": 0.4362472315795351,
|
|
"grad_norm": 6.83722298414795,
|
|
"learning_rate": 3.466828163951747e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4236,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11720
|
|
},
|
|
{
|
|
"epoch": 0.4366194561798589,
|
|
"grad_norm": 6.549996760425483,
|
|
"learning_rate": 3.46383169860756e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11730
|
|
},
|
|
{
|
|
"epoch": 0.43699168078018275,
|
|
"grad_norm": 6.381855935787022,
|
|
"learning_rate": 3.460833605991405e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3995,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11740
|
|
},
|
|
{
|
|
"epoch": 0.4373639053805066,
|
|
"grad_norm": 8.835793213265797,
|
|
"learning_rate": 3.4578338911650713e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4543,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 11750
|
|
},
|
|
{
|
|
"epoch": 0.43773612998083045,
|
|
"grad_norm": 7.211064763232928,
|
|
"learning_rate": 3.454832559193085e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 11760
|
|
},
|
|
{
|
|
"epoch": 0.43810835458115427,
|
|
"grad_norm": 7.427019036960049,
|
|
"learning_rate": 3.4518296151427036e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3817,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 11770
|
|
},
|
|
{
|
|
"epoch": 0.4384805791814781,
|
|
"grad_norm": 8.360298423422272,
|
|
"learning_rate": 3.448825064083907e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4002,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11780
|
|
},
|
|
{
|
|
"epoch": 0.4388528037818019,
|
|
"grad_norm": 7.270403516623677,
|
|
"learning_rate": 3.445818911089387e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11790
|
|
},
|
|
{
|
|
"epoch": 0.4392250283821258,
|
|
"grad_norm": 7.1425596002116025,
|
|
"learning_rate": 3.4428111612345413e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4104,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11800
|
|
},
|
|
{
|
|
"epoch": 0.4395972529824496,
|
|
"grad_norm": 5.8997381703592815,
|
|
"learning_rate": 3.439801819597462e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.984375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4224,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11810
|
|
},
|
|
{
|
|
"epoch": 0.43996947758277344,
|
|
"grad_norm": 6.707237155153718,
|
|
"learning_rate": 3.4367908912589307e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.401,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 11820
|
|
},
|
|
{
|
|
"epoch": 0.44034170218309726,
|
|
"grad_norm": 6.40447319144001,
|
|
"learning_rate": 3.4337783813024057e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4001,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11830
|
|
},
|
|
{
|
|
"epoch": 0.44071392678342114,
|
|
"grad_norm": 7.099287115885227,
|
|
"learning_rate": 3.430764294814017e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11840
|
|
},
|
|
{
|
|
"epoch": 0.44108615138374496,
|
|
"grad_norm": 6.496456666515634,
|
|
"learning_rate": 3.4277486368825557e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4029,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 11850
|
|
},
|
|
{
|
|
"epoch": 0.4414583759840688,
|
|
"grad_norm": 7.384739827979538,
|
|
"learning_rate": 3.4247314125994677e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3715,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11860
|
|
},
|
|
{
|
|
"epoch": 0.4418306005843926,
|
|
"grad_norm": 6.6666287956414525,
|
|
"learning_rate": 3.421712627058841e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11870
|
|
},
|
|
{
|
|
"epoch": 0.44220282518471643,
|
|
"grad_norm": 7.4495703826733966,
|
|
"learning_rate": 3.4186922853573996e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11880
|
|
},
|
|
{
|
|
"epoch": 0.4425750497850403,
|
|
"grad_norm": 6.9904788750723785,
|
|
"learning_rate": 3.415670392594498e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 11890
|
|
},
|
|
{
|
|
"epoch": 0.44294727438536413,
|
|
"grad_norm": 6.822305064104931,
|
|
"learning_rate": 3.4126469538721053e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4153,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 11900
|
|
},
|
|
{
|
|
"epoch": 0.44331949898568795,
|
|
"grad_norm": 5.5241005504522,
|
|
"learning_rate": 3.409621974294804e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3796,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 11910
|
|
},
|
|
{
|
|
"epoch": 0.4436917235860118,
|
|
"grad_norm": 5.529353485573811,
|
|
"learning_rate": 3.4065954589697753e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4334,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11920
|
|
},
|
|
{
|
|
"epoch": 0.44406394818633566,
|
|
"grad_norm": 5.332649522506943,
|
|
"learning_rate": 3.403567413006796e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4109,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 11930
|
|
},
|
|
{
|
|
"epoch": 0.4444361727866595,
|
|
"grad_norm": 6.713293980660568,
|
|
"learning_rate": 3.400537841518224e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.3946,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 11940
|
|
},
|
|
{
|
|
"epoch": 0.4448083973869833,
|
|
"grad_norm": 6.170190963454702,
|
|
"learning_rate": 3.3975067496189963e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11950
|
|
},
|
|
{
|
|
"epoch": 0.4451806219873071,
|
|
"grad_norm": 6.6194462032408445,
|
|
"learning_rate": 3.394474142426615e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11960
|
|
},
|
|
{
|
|
"epoch": 0.445552846587631,
|
|
"grad_norm": 7.403814839583554,
|
|
"learning_rate": 3.39144002506114e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4195,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 11970
|
|
},
|
|
{
|
|
"epoch": 0.4459250711879548,
|
|
"grad_norm": 7.5338988970094185,
|
|
"learning_rate": 3.388404402645182e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 11980
|
|
},
|
|
{
|
|
"epoch": 0.44629729578827865,
|
|
"grad_norm": 6.828200931367152,
|
|
"learning_rate": 3.3853672803038916e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 11990
|
|
},
|
|
{
|
|
"epoch": 0.44666952038860247,
|
|
"grad_norm": 7.90646150340077,
|
|
"learning_rate": 3.382328663164954e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12000
|
|
},
|
|
{
|
|
"epoch": 0.4470417449889263,
|
|
"grad_norm": 6.108188950900395,
|
|
"learning_rate": 3.379288556358574e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4114,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12010
|
|
},
|
|
{
|
|
"epoch": 0.44741396958925017,
|
|
"grad_norm": 6.422887604509636,
|
|
"learning_rate": 3.376246965017476e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12020
|
|
},
|
|
{
|
|
"epoch": 0.447786194189574,
|
|
"grad_norm": 6.536002560798968,
|
|
"learning_rate": 3.3732038942768895e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3937,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12030
|
|
},
|
|
{
|
|
"epoch": 0.4481584187898978,
|
|
"grad_norm": 8.249634671000358,
|
|
"learning_rate": 3.3701593492745386e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -852.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4255,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 12040
|
|
},
|
|
{
|
|
"epoch": 0.44853064339022164,
|
|
"grad_norm": 6.398874136997926,
|
|
"learning_rate": 3.36711333515064e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.385,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12050
|
|
},
|
|
{
|
|
"epoch": 0.4489028679905455,
|
|
"grad_norm": 5.5555331395429715,
|
|
"learning_rate": 3.364065857047891e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3981,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12060
|
|
},
|
|
{
|
|
"epoch": 0.44927509259086934,
|
|
"grad_norm": 6.904665580818742,
|
|
"learning_rate": 3.361016920111458e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3904,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12070
|
|
},
|
|
{
|
|
"epoch": 0.44964731719119316,
|
|
"grad_norm": 7.378094973235853,
|
|
"learning_rate": 3.3579665294889714e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 12080
|
|
},
|
|
{
|
|
"epoch": 0.450019541791517,
|
|
"grad_norm": 6.026805267166445,
|
|
"learning_rate": 3.354914690330517e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12090
|
|
},
|
|
{
|
|
"epoch": 0.45039176639184086,
|
|
"grad_norm": 6.202219125325113,
|
|
"learning_rate": 3.3518614077886246e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12100
|
|
},
|
|
{
|
|
"epoch": 0.4507639909921647,
|
|
"grad_norm": 6.473420474759876,
|
|
"learning_rate": 3.348806687018262e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 12110
|
|
},
|
|
{
|
|
"epoch": 0.4511362155924885,
|
|
"grad_norm": 8.382031797054704,
|
|
"learning_rate": 3.3457505331768254e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4206,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12120
|
|
},
|
|
{
|
|
"epoch": 0.45150844019281233,
|
|
"grad_norm": 7.2565020412934125,
|
|
"learning_rate": 3.3426929514241294e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 12130
|
|
},
|
|
{
|
|
"epoch": 0.45188066479313616,
|
|
"grad_norm": 6.622587653739626,
|
|
"learning_rate": 3.3396339469224e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.427,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12140
|
|
},
|
|
{
|
|
"epoch": 0.45225288939346003,
|
|
"grad_norm": 6.608473419577042,
|
|
"learning_rate": 3.336573524836266e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 12150
|
|
},
|
|
{
|
|
"epoch": 0.45262511399378386,
|
|
"grad_norm": 6.271214708674291,
|
|
"learning_rate": 3.3335116903327485e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3946,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12160
|
|
},
|
|
{
|
|
"epoch": 0.4529973385941077,
|
|
"grad_norm": 6.734927279394665,
|
|
"learning_rate": 3.330448448581254e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.4108,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 12170
|
|
},
|
|
{
|
|
"epoch": 0.4533695631944315,
|
|
"grad_norm": 8.308921121148376,
|
|
"learning_rate": 3.327383804753563e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12180
|
|
},
|
|
{
|
|
"epoch": 0.4537417877947554,
|
|
"grad_norm": 6.725740434032084,
|
|
"learning_rate": 3.3243177640238263e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3912,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 12190
|
|
},
|
|
{
|
|
"epoch": 0.4541140123950792,
|
|
"grad_norm": 8.237353307057845,
|
|
"learning_rate": 3.32125033156855e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3975,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12200
|
|
},
|
|
{
|
|
"epoch": 0.454486236995403,
|
|
"grad_norm": 8.459310165194896,
|
|
"learning_rate": 3.3181815125665943e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3973,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 12210
|
|
},
|
|
{
|
|
"epoch": 0.45485846159572685,
|
|
"grad_norm": 6.586399831954476,
|
|
"learning_rate": 3.315111312199154e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12220
|
|
},
|
|
{
|
|
"epoch": 0.45523068619605067,
|
|
"grad_norm": 6.533690318755923,
|
|
"learning_rate": 3.312039735649761e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3963,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12230
|
|
},
|
|
{
|
|
"epoch": 0.45560291079637455,
|
|
"grad_norm": 7.267672366151556,
|
|
"learning_rate": 3.308966788104271e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12240
|
|
},
|
|
{
|
|
"epoch": 0.45597513539669837,
|
|
"grad_norm": 7.944603578369851,
|
|
"learning_rate": 3.305892474750849e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.395,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12250
|
|
},
|
|
{
|
|
"epoch": 0.4563473599970222,
|
|
"grad_norm": 7.915772866741958,
|
|
"learning_rate": 3.3028168007799736e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12260
|
|
},
|
|
{
|
|
"epoch": 0.456719584597346,
|
|
"grad_norm": 6.9695868313508385,
|
|
"learning_rate": 3.299739771384413e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4135,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12270
|
|
},
|
|
{
|
|
"epoch": 0.4570918091976699,
|
|
"grad_norm": 6.422782277531488,
|
|
"learning_rate": 3.296661391759229e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4101,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12280
|
|
},
|
|
{
|
|
"epoch": 0.4574640337979937,
|
|
"grad_norm": 6.448756283303974,
|
|
"learning_rate": 3.2935816671017624e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3741,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12290
|
|
},
|
|
{
|
|
"epoch": 0.45783625839831754,
|
|
"grad_norm": 10.187371364392193,
|
|
"learning_rate": 3.290500602611622e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3949,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12300
|
|
},
|
|
{
|
|
"epoch": 0.45820848299864136,
|
|
"grad_norm": 6.723847471910078,
|
|
"learning_rate": 3.2874182034906816e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4104,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12310
|
|
},
|
|
{
|
|
"epoch": 0.45858070759896524,
|
|
"grad_norm": 5.842304218631227,
|
|
"learning_rate": 3.2843344749430666e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12320
|
|
},
|
|
{
|
|
"epoch": 0.45895293219928907,
|
|
"grad_norm": 7.247216546077987,
|
|
"learning_rate": 3.28124942217515e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4147,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12330
|
|
},
|
|
{
|
|
"epoch": 0.4593251567996129,
|
|
"grad_norm": 6.468587131905131,
|
|
"learning_rate": 3.2781630503955344e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4018,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 12340
|
|
},
|
|
{
|
|
"epoch": 0.4596973813999367,
|
|
"grad_norm": 7.347882039888926,
|
|
"learning_rate": 3.275075364815056e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12350
|
|
},
|
|
{
|
|
"epoch": 0.46006960600026053,
|
|
"grad_norm": 6.28618581314192,
|
|
"learning_rate": 3.271986370646764e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12360
|
|
},
|
|
{
|
|
"epoch": 0.4604418306005844,
|
|
"grad_norm": 7.229050504622546,
|
|
"learning_rate": 3.2688960731059204e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3861,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12370
|
|
},
|
|
{
|
|
"epoch": 0.46081405520090823,
|
|
"grad_norm": 7.817093753983414,
|
|
"learning_rate": 3.2658044774099875e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12380
|
|
},
|
|
{
|
|
"epoch": 0.46118627980123206,
|
|
"grad_norm": 7.406565116489547,
|
|
"learning_rate": 3.262711588778616e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3892,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12390
|
|
},
|
|
{
|
|
"epoch": 0.4615585044015559,
|
|
"grad_norm": 7.6850109207771355,
|
|
"learning_rate": 3.259617412433644e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4007,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12400
|
|
},
|
|
{
|
|
"epoch": 0.46193072900187976,
|
|
"grad_norm": 7.389945431672609,
|
|
"learning_rate": 3.25652195359908e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3959,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12410
|
|
},
|
|
{
|
|
"epoch": 0.4623029536022036,
|
|
"grad_norm": 7.478573413190836,
|
|
"learning_rate": 3.2534252175010994e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.401,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12420
|
|
},
|
|
{
|
|
"epoch": 0.4626751782025274,
|
|
"grad_norm": 5.778871050114823,
|
|
"learning_rate": 3.2503272093680347e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4077,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 12430
|
|
},
|
|
{
|
|
"epoch": 0.4630474028028512,
|
|
"grad_norm": 6.312710891978031,
|
|
"learning_rate": 3.247227934430364e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3658,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12440
|
|
},
|
|
{
|
|
"epoch": 0.46341962740317505,
|
|
"grad_norm": 6.399813974777586,
|
|
"learning_rate": 3.2441273979207074e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12450
|
|
},
|
|
{
|
|
"epoch": 0.46379185200349893,
|
|
"grad_norm": 8.417638450528834,
|
|
"learning_rate": 3.2410256050738107e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3943,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 12460
|
|
},
|
|
{
|
|
"epoch": 0.46416407660382275,
|
|
"grad_norm": 6.679393205720345,
|
|
"learning_rate": 3.237922561126545e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 12470
|
|
},
|
|
{
|
|
"epoch": 0.4645363012041466,
|
|
"grad_norm": 7.416021315568357,
|
|
"learning_rate": 3.23481827131789e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4212,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 12480
|
|
},
|
|
{
|
|
"epoch": 0.4649085258044704,
|
|
"grad_norm": 5.853262916998044,
|
|
"learning_rate": 3.2317127408889327e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12490
|
|
},
|
|
{
|
|
"epoch": 0.4652807504047943,
|
|
"grad_norm": 6.180046167177449,
|
|
"learning_rate": 3.228605975082851e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.3949,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 12500
|
|
},
|
|
{
|
|
"epoch": 0.4656529750051181,
|
|
"grad_norm": 6.963126883549509,
|
|
"learning_rate": 3.225497979144911e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12510
|
|
},
|
|
{
|
|
"epoch": 0.4660251996054419,
|
|
"grad_norm": 6.71359524190679,
|
|
"learning_rate": 3.222388758322454e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4238,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 12520
|
|
},
|
|
{
|
|
"epoch": 0.46639742420576574,
|
|
"grad_norm": 7.333288803890293,
|
|
"learning_rate": 3.219278317864891e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4115,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12530
|
|
},
|
|
{
|
|
"epoch": 0.4667696488060896,
|
|
"grad_norm": 7.407332493264781,
|
|
"learning_rate": 3.2161666630236913e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.377,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12540
|
|
},
|
|
{
|
|
"epoch": 0.46714187340641344,
|
|
"grad_norm": 7.183185837461339,
|
|
"learning_rate": 3.213053799052373e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3813,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12550
|
|
},
|
|
{
|
|
"epoch": 0.46751409800673727,
|
|
"grad_norm": 5.660030615473503,
|
|
"learning_rate": 3.209939731206499e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 12560
|
|
},
|
|
{
|
|
"epoch": 0.4678863226070611,
|
|
"grad_norm": 7.202827749093596,
|
|
"learning_rate": 3.206824464743662e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12570
|
|
},
|
|
{
|
|
"epoch": 0.4682585472073849,
|
|
"grad_norm": 8.14033245176476,
|
|
"learning_rate": 3.2037080049234795e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4085,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12580
|
|
},
|
|
{
|
|
"epoch": 0.4686307718077088,
|
|
"grad_norm": 5.983774166166898,
|
|
"learning_rate": 3.2005903570075826e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -4.0,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.399,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 12590
|
|
},
|
|
{
|
|
"epoch": 0.4690029964080326,
|
|
"grad_norm": 5.790467169749596,
|
|
"learning_rate": 3.197471526259611e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12600
|
|
},
|
|
{
|
|
"epoch": 0.46937522100835644,
|
|
"grad_norm": 6.592596671432231,
|
|
"learning_rate": 3.194351517945197e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -4.125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12610
|
|
},
|
|
{
|
|
"epoch": 0.46974744560868026,
|
|
"grad_norm": 6.451172522128122,
|
|
"learning_rate": 3.1912303373319654e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3936,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12620
|
|
},
|
|
{
|
|
"epoch": 0.47011967020900414,
|
|
"grad_norm": 6.224399086366324,
|
|
"learning_rate": 3.1881079896895187e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3811,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12630
|
|
},
|
|
{
|
|
"epoch": 0.47049189480932796,
|
|
"grad_norm": 6.4066798365096345,
|
|
"learning_rate": 3.184984480289429e-07,
|
|
"logits/chosen": -4.15625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3698,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 12640
|
|
},
|
|
{
|
|
"epoch": 0.4708641194096518,
|
|
"grad_norm": 8.369364244520712,
|
|
"learning_rate": 3.18185981440523e-07,
|
|
"logits/chosen": -4.1875,
|
|
"logits/rejected": -4.0625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4018,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12650
|
|
},
|
|
{
|
|
"epoch": 0.4712363440099756,
|
|
"grad_norm": 9.416386628293159,
|
|
"learning_rate": 3.1787339973124087e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12660
|
|
},
|
|
{
|
|
"epoch": 0.47160856861029943,
|
|
"grad_norm": 9.161174310684117,
|
|
"learning_rate": 3.1756070342883955e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4063,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12670
|
|
},
|
|
{
|
|
"epoch": 0.4719807932106233,
|
|
"grad_norm": 7.155634529061662,
|
|
"learning_rate": 3.1724789306125556e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 12680
|
|
},
|
|
{
|
|
"epoch": 0.47235301781094713,
|
|
"grad_norm": 7.030629563135893,
|
|
"learning_rate": 3.1693496915661776e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12690
|
|
},
|
|
{
|
|
"epoch": 0.47272524241127095,
|
|
"grad_norm": 6.835152228167354,
|
|
"learning_rate": 3.1662193224324717e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -6.75,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 12700
|
|
},
|
|
{
|
|
"epoch": 0.4730974670115948,
|
|
"grad_norm": 6.4158582857575945,
|
|
"learning_rate": 3.163087828496553e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4105,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 12710
|
|
},
|
|
{
|
|
"epoch": 0.47346969161191865,
|
|
"grad_norm": 6.742948599776619,
|
|
"learning_rate": 3.1599552150454357e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3826,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.5,
|
|
"step": 12720
|
|
},
|
|
{
|
|
"epoch": 0.4738419162122425,
|
|
"grad_norm": 6.958399975551548,
|
|
"learning_rate": 3.1568214873680245e-07,
|
|
"logits/chosen": -4.09375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12730
|
|
},
|
|
{
|
|
"epoch": 0.4742141408125663,
|
|
"grad_norm": 6.6223073934919885,
|
|
"learning_rate": 3.1536866507551047e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 12740
|
|
},
|
|
{
|
|
"epoch": 0.4745863654128901,
|
|
"grad_norm": 6.125850251282852,
|
|
"learning_rate": 3.150550710499336e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3862,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12750
|
|
},
|
|
{
|
|
"epoch": 0.474958590013214,
|
|
"grad_norm": 6.46113172501067,
|
|
"learning_rate": 3.1474136718952384e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 12760
|
|
},
|
|
{
|
|
"epoch": 0.4753308146135378,
|
|
"grad_norm": 7.247077135296653,
|
|
"learning_rate": 3.144275540239189e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3977,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12770
|
|
},
|
|
{
|
|
"epoch": 0.47570303921386164,
|
|
"grad_norm": 7.6922387415433695,
|
|
"learning_rate": 3.1411363208294077e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.396,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 12780
|
|
},
|
|
{
|
|
"epoch": 0.47607526381418547,
|
|
"grad_norm": 8.55928770294166,
|
|
"learning_rate": 3.137996018965955e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4304,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12790
|
|
},
|
|
{
|
|
"epoch": 0.4764474884145093,
|
|
"grad_norm": 7.012048944752695,
|
|
"learning_rate": 3.1348546399507137e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3986,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12800
|
|
},
|
|
{
|
|
"epoch": 0.47681971301483317,
|
|
"grad_norm": 6.83122068372055,
|
|
"learning_rate": 3.131712189087389e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.3944,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12810
|
|
},
|
|
{
|
|
"epoch": 0.477191937615157,
|
|
"grad_norm": 7.101183289041492,
|
|
"learning_rate": 3.128568671681494e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.8125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 12820
|
|
},
|
|
{
|
|
"epoch": 0.4775641622154808,
|
|
"grad_norm": 7.248250071154567,
|
|
"learning_rate": 3.1254240930403424e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12830
|
|
},
|
|
{
|
|
"epoch": 0.47793638681580464,
|
|
"grad_norm": 7.608874465551103,
|
|
"learning_rate": 3.122278458473042e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3933,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12840
|
|
},
|
|
{
|
|
"epoch": 0.4783086114161285,
|
|
"grad_norm": 6.822411957647919,
|
|
"learning_rate": 3.1191317732904803e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 12850
|
|
},
|
|
{
|
|
"epoch": 0.47868083601645234,
|
|
"grad_norm": 5.648650765267038,
|
|
"learning_rate": 3.11598404280532e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3919,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12860
|
|
},
|
|
{
|
|
"epoch": 0.47905306061677616,
|
|
"grad_norm": 7.300180660227479,
|
|
"learning_rate": 3.112835272331989e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 12870
|
|
},
|
|
{
|
|
"epoch": 0.4794252852171,
|
|
"grad_norm": 7.604791458334221,
|
|
"learning_rate": 3.1096854671866696e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12880
|
|
},
|
|
{
|
|
"epoch": 0.4797975098174238,
|
|
"grad_norm": 7.730657179069855,
|
|
"learning_rate": 3.106534632687293e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4108,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12890
|
|
},
|
|
{
|
|
"epoch": 0.4801697344177477,
|
|
"grad_norm": 5.049757801165113,
|
|
"learning_rate": 3.103382774153526e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3768,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 12900
|
|
},
|
|
{
|
|
"epoch": 0.4805419590180715,
|
|
"grad_norm": 7.108258672161911,
|
|
"learning_rate": 3.1002298969067676e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4166,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 12910
|
|
},
|
|
{
|
|
"epoch": 0.48091418361839533,
|
|
"grad_norm": 7.157796704977386,
|
|
"learning_rate": 3.097076006270132e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 12920
|
|
},
|
|
{
|
|
"epoch": 0.48128640821871915,
|
|
"grad_norm": 7.286462479743474,
|
|
"learning_rate": 3.093921107568449e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 12930
|
|
},
|
|
{
|
|
"epoch": 0.48165863281904303,
|
|
"grad_norm": 5.574117694446025,
|
|
"learning_rate": 3.0907652061282467e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.4075,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12940
|
|
},
|
|
{
|
|
"epoch": 0.48203085741936685,
|
|
"grad_norm": 6.698436758568989,
|
|
"learning_rate": 3.087608307277749e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -840.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.5625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 12950
|
|
},
|
|
{
|
|
"epoch": 0.4824030820196907,
|
|
"grad_norm": 6.346331020720606,
|
|
"learning_rate": 3.0844504163468634e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.396,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 12960
|
|
},
|
|
{
|
|
"epoch": 0.4827753066200145,
|
|
"grad_norm": 6.974106938063188,
|
|
"learning_rate": 3.081291538667169e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12970
|
|
},
|
|
{
|
|
"epoch": 0.4831475312203384,
|
|
"grad_norm": 7.141521333823262,
|
|
"learning_rate": 3.0781316795719165e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 12980
|
|
},
|
|
{
|
|
"epoch": 0.4835197558206622,
|
|
"grad_norm": 7.11804443438213,
|
|
"learning_rate": 3.0749708443960076e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 12990
|
|
},
|
|
{
|
|
"epoch": 0.483891980420986,
|
|
"grad_norm": 7.516665197812263,
|
|
"learning_rate": 3.071809038475997e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4009,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13000
|
|
},
|
|
{
|
|
"epoch": 0.48426420502130985,
|
|
"grad_norm": 6.184474779879333,
|
|
"learning_rate": 3.0686462671500744e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4356,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13010
|
|
},
|
|
{
|
|
"epoch": 0.48463642962163367,
|
|
"grad_norm": 6.915745327325248,
|
|
"learning_rate": 3.065482535758063e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4049,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13020
|
|
},
|
|
{
|
|
"epoch": 0.48500865422195755,
|
|
"grad_norm": 7.387016215069922,
|
|
"learning_rate": 3.0623178496414025e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13030
|
|
},
|
|
{
|
|
"epoch": 0.48538087882228137,
|
|
"grad_norm": 7.725064138441932,
|
|
"learning_rate": 3.0591522141431495e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4141,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13040
|
|
},
|
|
{
|
|
"epoch": 0.4857531034226052,
|
|
"grad_norm": 6.55140628464621,
|
|
"learning_rate": 3.0559856346079605e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13050
|
|
},
|
|
{
|
|
"epoch": 0.486125328022929,
|
|
"grad_norm": 5.615100268044686,
|
|
"learning_rate": 3.052818116382086e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3974,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13060
|
|
},
|
|
{
|
|
"epoch": 0.4864975526232529,
|
|
"grad_norm": 6.1782284478168705,
|
|
"learning_rate": 3.0496496648133623e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13070
|
|
},
|
|
{
|
|
"epoch": 0.4868697772235767,
|
|
"grad_norm": 6.206074714468112,
|
|
"learning_rate": 3.0464802852512e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13080
|
|
},
|
|
{
|
|
"epoch": 0.48724200182390054,
|
|
"grad_norm": 7.68620887123333,
|
|
"learning_rate": 3.0433099830465784e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13090
|
|
},
|
|
{
|
|
"epoch": 0.48761422642422436,
|
|
"grad_norm": 5.917807556809307,
|
|
"learning_rate": 3.040138763552034e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.369,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 13100
|
|
},
|
|
{
|
|
"epoch": 0.48798645102454824,
|
|
"grad_norm": 6.613768785294606,
|
|
"learning_rate": 3.0369666321216494e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13110
|
|
},
|
|
{
|
|
"epoch": 0.48835867562487206,
|
|
"grad_norm": 7.195769512643943,
|
|
"learning_rate": 3.03379359411105e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4033,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13120
|
|
},
|
|
{
|
|
"epoch": 0.4887309002251959,
|
|
"grad_norm": 7.174768866010633,
|
|
"learning_rate": 3.0306196548773914e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4018,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13130
|
|
},
|
|
{
|
|
"epoch": 0.4891031248255197,
|
|
"grad_norm": 6.6025040965280075,
|
|
"learning_rate": 3.027444819779351e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3955,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13140
|
|
},
|
|
{
|
|
"epoch": 0.48947534942584353,
|
|
"grad_norm": 6.00957430106668,
|
|
"learning_rate": 3.024269094177116e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3708,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13150
|
|
},
|
|
{
|
|
"epoch": 0.4898475740261674,
|
|
"grad_norm": 6.788098665551054,
|
|
"learning_rate": 3.0210924834323794e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 13160
|
|
},
|
|
{
|
|
"epoch": 0.49021979862649123,
|
|
"grad_norm": 8.598115567201967,
|
|
"learning_rate": 3.0179149929083293e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4274,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13170
|
|
},
|
|
{
|
|
"epoch": 0.49059202322681505,
|
|
"grad_norm": 6.817543118577306,
|
|
"learning_rate": 3.014736627969636e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4015,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13180
|
|
},
|
|
{
|
|
"epoch": 0.4909642478271389,
|
|
"grad_norm": 7.388337812803533,
|
|
"learning_rate": 3.0115573939824507e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.401,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13190
|
|
},
|
|
{
|
|
"epoch": 0.49133647242746276,
|
|
"grad_norm": 7.023634134644685,
|
|
"learning_rate": 3.0083772963143866e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4136,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13200
|
|
},
|
|
{
|
|
"epoch": 0.4917086970277866,
|
|
"grad_norm": 6.716929849197618,
|
|
"learning_rate": 3.00519634033452e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13210
|
|
},
|
|
{
|
|
"epoch": 0.4920809216281104,
|
|
"grad_norm": 7.191128407011848,
|
|
"learning_rate": 3.002014531413374e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4081,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13220
|
|
},
|
|
{
|
|
"epoch": 0.4924531462284342,
|
|
"grad_norm": 5.91530384434546,
|
|
"learning_rate": 2.99883187492291e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3977,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13230
|
|
},
|
|
{
|
|
"epoch": 0.49282537082875805,
|
|
"grad_norm": 6.612534766142053,
|
|
"learning_rate": 2.995648376236524e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.411,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13240
|
|
},
|
|
{
|
|
"epoch": 0.4931975954290819,
|
|
"grad_norm": 6.288650999894333,
|
|
"learning_rate": 2.992464040729031e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3965,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13250
|
|
},
|
|
{
|
|
"epoch": 0.49356982002940575,
|
|
"grad_norm": 6.094028678430905,
|
|
"learning_rate": 2.989278873776661e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13260
|
|
},
|
|
{
|
|
"epoch": 0.49394204462972957,
|
|
"grad_norm": 6.774950089057636,
|
|
"learning_rate": 2.986092880757046e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13270
|
|
},
|
|
{
|
|
"epoch": 0.4943142692300534,
|
|
"grad_norm": 5.594535983649743,
|
|
"learning_rate": 2.9829060670492134e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13280
|
|
},
|
|
{
|
|
"epoch": 0.49468649383037727,
|
|
"grad_norm": 7.179341750949797,
|
|
"learning_rate": 2.9797184380335763e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 13290
|
|
},
|
|
{
|
|
"epoch": 0.4950587184307011,
|
|
"grad_norm": 7.197609371979025,
|
|
"learning_rate": 2.976529999091924e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3807,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 13300
|
|
},
|
|
{
|
|
"epoch": 0.4954309430310249,
|
|
"grad_norm": 7.770079733716399,
|
|
"learning_rate": 2.9733407556074146e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13310
|
|
},
|
|
{
|
|
"epoch": 0.49580316763134874,
|
|
"grad_norm": 7.459434857307718,
|
|
"learning_rate": 2.9701507129645625e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4058,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 13320
|
|
},
|
|
{
|
|
"epoch": 0.4961753922316726,
|
|
"grad_norm": 6.885348980640635,
|
|
"learning_rate": 2.966959876549233e-07,
|
|
"logits/chosen": -4.0625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13330
|
|
},
|
|
{
|
|
"epoch": 0.49654761683199644,
|
|
"grad_norm": 5.702821003914519,
|
|
"learning_rate": 2.9637682517486284e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3975,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13340
|
|
},
|
|
{
|
|
"epoch": 0.49691984143232026,
|
|
"grad_norm": 6.317335820451346,
|
|
"learning_rate": 2.9605758439512874e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13350
|
|
},
|
|
{
|
|
"epoch": 0.4972920660326441,
|
|
"grad_norm": 7.74264513029993,
|
|
"learning_rate": 2.957382658547066e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13360
|
|
},
|
|
{
|
|
"epoch": 0.4976642906329679,
|
|
"grad_norm": 7.867522704818632,
|
|
"learning_rate": 2.9541887009271354e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.395,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13370
|
|
},
|
|
{
|
|
"epoch": 0.4980365152332918,
|
|
"grad_norm": 8.242107804680115,
|
|
"learning_rate": 2.95099397648397e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.425,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13380
|
|
},
|
|
{
|
|
"epoch": 0.4984087398336156,
|
|
"grad_norm": 7.1205129326550445,
|
|
"learning_rate": 2.947798490611339e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13390
|
|
},
|
|
{
|
|
"epoch": 0.49878096443393943,
|
|
"grad_norm": 7.065382019289934,
|
|
"learning_rate": 2.9446022487042966e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13400
|
|
},
|
|
{
|
|
"epoch": 0.49915318903426326,
|
|
"grad_norm": 7.4351977617952505,
|
|
"learning_rate": 2.9414052561591727e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3817,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 13410
|
|
},
|
|
{
|
|
"epoch": 0.49952541363458713,
|
|
"grad_norm": 5.5352520224951665,
|
|
"learning_rate": 2.9382075183735675e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3943,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13420
|
|
},
|
|
{
|
|
"epoch": 0.49989763823491096,
|
|
"grad_norm": 6.8536320675346545,
|
|
"learning_rate": 2.9350090407463363e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3851,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13430
|
|
},
|
|
{
|
|
"epoch": 0.5002698628352348,
|
|
"grad_norm": 6.755779429817708,
|
|
"learning_rate": 2.9318098286775857e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.397,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 13440
|
|
},
|
|
{
|
|
"epoch": 0.5006420874355586,
|
|
"grad_norm": 7.788114534140766,
|
|
"learning_rate": 2.92860988756866e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 13450
|
|
},
|
|
{
|
|
"epoch": 0.5010143120358824,
|
|
"grad_norm": 6.292478520723932,
|
|
"learning_rate": 2.9254092228221365e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3732,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13460
|
|
},
|
|
{
|
|
"epoch": 0.5013865366362062,
|
|
"grad_norm": 7.493676941016451,
|
|
"learning_rate": 2.922207839841814e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13470
|
|
},
|
|
{
|
|
"epoch": 0.5017587612365301,
|
|
"grad_norm": 6.809355565011525,
|
|
"learning_rate": 2.919005744032702e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13480
|
|
},
|
|
{
|
|
"epoch": 0.502130985836854,
|
|
"grad_norm": 6.627163938427665,
|
|
"learning_rate": 2.915802940801016e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4112,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13490
|
|
},
|
|
{
|
|
"epoch": 0.5025032104371778,
|
|
"grad_norm": 6.016897343330382,
|
|
"learning_rate": 2.9125994355541637e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3713,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 13500
|
|
},
|
|
{
|
|
"epoch": 0.5028754350375017,
|
|
"grad_norm": 6.365970158861036,
|
|
"learning_rate": 2.9093952337007404e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13510
|
|
},
|
|
{
|
|
"epoch": 0.5032476596378255,
|
|
"grad_norm": 8.029625825618265,
|
|
"learning_rate": 2.9061903406505153e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3976,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13520
|
|
},
|
|
{
|
|
"epoch": 0.5036198842381493,
|
|
"grad_norm": 7.1545480740689555,
|
|
"learning_rate": 2.9029847618144243e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4185,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 13530
|
|
},
|
|
{
|
|
"epoch": 0.5039921088384731,
|
|
"grad_norm": 7.353124418388507,
|
|
"learning_rate": 2.8997785026045643e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.405,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13540
|
|
},
|
|
{
|
|
"epoch": 0.5043643334387969,
|
|
"grad_norm": 7.316713510272069,
|
|
"learning_rate": 2.8965715684341767e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4022,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13550
|
|
},
|
|
{
|
|
"epoch": 0.5047365580391208,
|
|
"grad_norm": 7.058632781676106,
|
|
"learning_rate": 2.893363964717647e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.399,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 13560
|
|
},
|
|
{
|
|
"epoch": 0.5051087826394446,
|
|
"grad_norm": 6.332378999553435,
|
|
"learning_rate": 2.8901556968704875e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4076,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 13570
|
|
},
|
|
{
|
|
"epoch": 0.5054810072397685,
|
|
"grad_norm": 6.074773691196653,
|
|
"learning_rate": 2.886946770309333e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3796,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13580
|
|
},
|
|
{
|
|
"epoch": 0.5058532318400923,
|
|
"grad_norm": 6.658932415531432,
|
|
"learning_rate": 2.8837371904519315e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3766,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 13590
|
|
},
|
|
{
|
|
"epoch": 0.5062254564404162,
|
|
"grad_norm": 6.492281038245233,
|
|
"learning_rate": 2.88052696271713e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4235,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 13600
|
|
},
|
|
{
|
|
"epoch": 0.50659768104074,
|
|
"grad_norm": 7.742537738080852,
|
|
"learning_rate": 2.877316092524876e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4114,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13610
|
|
},
|
|
{
|
|
"epoch": 0.5069699056410638,
|
|
"grad_norm": 6.263659764925685,
|
|
"learning_rate": 2.8741045852961955e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3955,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13620
|
|
},
|
|
{
|
|
"epoch": 0.5073421302413876,
|
|
"grad_norm": 6.895568300609607,
|
|
"learning_rate": 2.870892446453193e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3736,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 13630
|
|
},
|
|
{
|
|
"epoch": 0.5077143548417115,
|
|
"grad_norm": 5.859603052284808,
|
|
"learning_rate": 2.867679681419039e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3688,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13640
|
|
},
|
|
{
|
|
"epoch": 0.5080865794420353,
|
|
"grad_norm": 9.379187898034214,
|
|
"learning_rate": 2.8644662956179617e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4071,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13650
|
|
},
|
|
{
|
|
"epoch": 0.5084588040423592,
|
|
"grad_norm": 7.905949934299075,
|
|
"learning_rate": 2.861252294475236e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13660
|
|
},
|
|
{
|
|
"epoch": 0.508831028642683,
|
|
"grad_norm": 6.590810777568883,
|
|
"learning_rate": 2.858037683417176e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13670
|
|
},
|
|
{
|
|
"epoch": 0.5092032532430069,
|
|
"grad_norm": 6.610770340072812,
|
|
"learning_rate": 2.8548224678711276e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4088,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13680
|
|
},
|
|
{
|
|
"epoch": 0.5095754778433307,
|
|
"grad_norm": 9.029996788591998,
|
|
"learning_rate": 2.8516066532654536e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4229,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13690
|
|
},
|
|
{
|
|
"epoch": 0.5099477024436545,
|
|
"grad_norm": 5.815031644042658,
|
|
"learning_rate": 2.8483902450295324e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13700
|
|
},
|
|
{
|
|
"epoch": 0.5103199270439783,
|
|
"grad_norm": 6.485090644735475,
|
|
"learning_rate": 2.8451732485937405e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 13710
|
|
},
|
|
{
|
|
"epoch": 0.5106921516443022,
|
|
"grad_norm": 7.22612702223559,
|
|
"learning_rate": 2.841955669389451e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.4169,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 13720
|
|
},
|
|
{
|
|
"epoch": 0.511064376244626,
|
|
"grad_norm": 5.461339772404787,
|
|
"learning_rate": 2.838737512849019e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3976,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 13730
|
|
},
|
|
{
|
|
"epoch": 0.5114366008449498,
|
|
"grad_norm": 7.843715442630194,
|
|
"learning_rate": 2.8355187844057736e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13740
|
|
},
|
|
{
|
|
"epoch": 0.5118088254452737,
|
|
"grad_norm": 6.019525727087064,
|
|
"learning_rate": 2.8322994894940127e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4294,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 13750
|
|
},
|
|
{
|
|
"epoch": 0.5121810500455976,
|
|
"grad_norm": 6.997909939865164,
|
|
"learning_rate": 2.829079633548987e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13760
|
|
},
|
|
{
|
|
"epoch": 0.5125532746459214,
|
|
"grad_norm": 7.916704178128282,
|
|
"learning_rate": 2.8258592220068966e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4036,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13770
|
|
},
|
|
{
|
|
"epoch": 0.5129254992462452,
|
|
"grad_norm": 6.4918474967217765,
|
|
"learning_rate": 2.8226382603048784e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3989,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 13780
|
|
},
|
|
{
|
|
"epoch": 0.513297723846569,
|
|
"grad_norm": 6.784608250586411,
|
|
"learning_rate": 2.819416753880999e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3974,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 13790
|
|
},
|
|
{
|
|
"epoch": 0.5136699484468928,
|
|
"grad_norm": 7.314264138600511,
|
|
"learning_rate": 2.8161947081742447e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4026,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13800
|
|
},
|
|
{
|
|
"epoch": 0.5140421730472167,
|
|
"grad_norm": 7.638481425612066,
|
|
"learning_rate": 2.812972128624511e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.374,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13810
|
|
},
|
|
{
|
|
"epoch": 0.5144143976475405,
|
|
"grad_norm": 6.036066085043316,
|
|
"learning_rate": 2.8097490206725976e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13820
|
|
},
|
|
{
|
|
"epoch": 0.5147866222478643,
|
|
"grad_norm": 7.954941224195803,
|
|
"learning_rate": 2.806525389760192e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4177,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 13830
|
|
},
|
|
{
|
|
"epoch": 0.5151588468481882,
|
|
"grad_norm": 7.183966520454452,
|
|
"learning_rate": 2.803301241329869e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4091,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 13840
|
|
},
|
|
{
|
|
"epoch": 0.5155310714485121,
|
|
"grad_norm": 7.353756568632751,
|
|
"learning_rate": 2.800076580825074e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -860.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.4057,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.875,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.375,
|
|
"step": 13850
|
|
},
|
|
{
|
|
"epoch": 0.5159032960488359,
|
|
"grad_norm": 7.208738485541649,
|
|
"learning_rate": 2.796851413690119e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4111,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -6.9375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13860
|
|
},
|
|
{
|
|
"epoch": 0.5162755206491597,
|
|
"grad_norm": 9.093439463885336,
|
|
"learning_rate": 2.7936257453701693e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3951,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 13870
|
|
},
|
|
{
|
|
"epoch": 0.5166477452494835,
|
|
"grad_norm": 7.040424780663929,
|
|
"learning_rate": 2.790399581311238e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 13880
|
|
},
|
|
{
|
|
"epoch": 0.5170199698498074,
|
|
"grad_norm": 7.600417458775632,
|
|
"learning_rate": 2.7871729269601763e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3903,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 13890
|
|
},
|
|
{
|
|
"epoch": 0.5173921944501312,
|
|
"grad_norm": 6.884600028697419,
|
|
"learning_rate": 2.7839457877646587e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3899,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 13900
|
|
},
|
|
{
|
|
"epoch": 0.517764419050455,
|
|
"grad_norm": 10.22852999333019,
|
|
"learning_rate": 2.780718169173184e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.383,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 13910
|
|
},
|
|
{
|
|
"epoch": 0.5181366436507788,
|
|
"grad_norm": 6.950639053313675,
|
|
"learning_rate": 2.7774900766350564e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4182,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13920
|
|
},
|
|
{
|
|
"epoch": 0.5185088682511028,
|
|
"grad_norm": 6.833538896333202,
|
|
"learning_rate": 2.7742615156003805e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 13930
|
|
},
|
|
{
|
|
"epoch": 0.5188810928514266,
|
|
"grad_norm": 6.6571832706566925,
|
|
"learning_rate": 2.7710324915200546e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3907,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13940
|
|
},
|
|
{
|
|
"epoch": 0.5192533174517504,
|
|
"grad_norm": 7.167563516608635,
|
|
"learning_rate": 2.767803009845756e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4007,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 13950
|
|
},
|
|
{
|
|
"epoch": 0.5196255420520742,
|
|
"grad_norm": 6.671067194460909,
|
|
"learning_rate": 2.764573076029935e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.391,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13960
|
|
},
|
|
{
|
|
"epoch": 0.519997766652398,
|
|
"grad_norm": 7.795349761395503,
|
|
"learning_rate": 2.7613426955258074e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.397,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13970
|
|
},
|
|
{
|
|
"epoch": 0.5203699912527219,
|
|
"grad_norm": 6.37317999804192,
|
|
"learning_rate": 2.758111873787341e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.96875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4256,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 13980
|
|
},
|
|
{
|
|
"epoch": 0.5207422158530457,
|
|
"grad_norm": 7.2562528599347,
|
|
"learning_rate": 2.754880616269248e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.395,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 13990
|
|
},
|
|
{
|
|
"epoch": 0.5211144404533695,
|
|
"grad_norm": 7.756180611686295,
|
|
"learning_rate": 2.75164892842698e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14000
|
|
},
|
|
{
|
|
"epoch": 0.5214866650536933,
|
|
"grad_norm": 7.732893465927635,
|
|
"learning_rate": 2.7484168157167095e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4002,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 14010
|
|
},
|
|
{
|
|
"epoch": 0.5218588896540173,
|
|
"grad_norm": 6.4544384984401395,
|
|
"learning_rate": 2.7451842835953326e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4214,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14020
|
|
},
|
|
{
|
|
"epoch": 0.5222311142543411,
|
|
"grad_norm": 6.266005517711986,
|
|
"learning_rate": 2.7419513375204485e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3773,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14030
|
|
},
|
|
{
|
|
"epoch": 0.5226033388546649,
|
|
"grad_norm": 11.275500592633746,
|
|
"learning_rate": 2.738717982950357e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14040
|
|
},
|
|
{
|
|
"epoch": 0.5229755634549887,
|
|
"grad_norm": 6.527329828685544,
|
|
"learning_rate": 2.7354842253440484e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3677,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14050
|
|
},
|
|
{
|
|
"epoch": 0.5233477880553126,
|
|
"grad_norm": 7.657426291985336,
|
|
"learning_rate": 2.7322500701611926e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14060
|
|
},
|
|
{
|
|
"epoch": 0.5237200126556364,
|
|
"grad_norm": 7.768624445080337,
|
|
"learning_rate": 2.7290155228621315e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14070
|
|
},
|
|
{
|
|
"epoch": 0.5240922372559602,
|
|
"grad_norm": 7.872329071912409,
|
|
"learning_rate": 2.7257805889078677e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4056,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14080
|
|
},
|
|
{
|
|
"epoch": 0.524464461856284,
|
|
"grad_norm": 7.621500018260968,
|
|
"learning_rate": 2.7225452737600574e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4137,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14090
|
|
},
|
|
{
|
|
"epoch": 0.524836686456608,
|
|
"grad_norm": 7.652697638611026,
|
|
"learning_rate": 2.7193095828810016e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14100
|
|
},
|
|
{
|
|
"epoch": 0.5252089110569318,
|
|
"grad_norm": 8.80122217400072,
|
|
"learning_rate": 2.7160735217336314e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4032,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14110
|
|
},
|
|
{
|
|
"epoch": 0.5255811356572556,
|
|
"grad_norm": 7.134968780280357,
|
|
"learning_rate": 2.71283709578151e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14120
|
|
},
|
|
{
|
|
"epoch": 0.5259533602575794,
|
|
"grad_norm": 11.615602324623964,
|
|
"learning_rate": 2.709600310488809e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3867,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14130
|
|
},
|
|
{
|
|
"epoch": 0.5263255848579033,
|
|
"grad_norm": 6.907050476052904,
|
|
"learning_rate": 2.706363171320313e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3883,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14140
|
|
},
|
|
{
|
|
"epoch": 0.5266978094582271,
|
|
"grad_norm": 8.243328116126921,
|
|
"learning_rate": 2.7031256837414e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4058,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14150
|
|
},
|
|
{
|
|
"epoch": 0.5270700340585509,
|
|
"grad_norm": 6.924720182341618,
|
|
"learning_rate": 2.6998878532180374e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3925,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14160
|
|
},
|
|
{
|
|
"epoch": 0.5274422586588747,
|
|
"grad_norm": 7.664041720695575,
|
|
"learning_rate": 2.6966496852167723e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4346,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14170
|
|
},
|
|
{
|
|
"epoch": 0.5278144832591986,
|
|
"grad_norm": 7.112058246797567,
|
|
"learning_rate": 2.693411185204721e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3988,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14180
|
|
},
|
|
{
|
|
"epoch": 0.5281867078595225,
|
|
"grad_norm": 6.353661323071403,
|
|
"learning_rate": 2.69017235864956e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14190
|
|
},
|
|
{
|
|
"epoch": 0.5285589324598463,
|
|
"grad_norm": 6.405481303968176,
|
|
"learning_rate": 2.686933211019517e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3901,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14200
|
|
},
|
|
{
|
|
"epoch": 0.5289311570601701,
|
|
"grad_norm": 7.216677967207719,
|
|
"learning_rate": 2.6836937477833637e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3937,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14210
|
|
},
|
|
{
|
|
"epoch": 0.529303381660494,
|
|
"grad_norm": 9.363533894484041,
|
|
"learning_rate": 2.680453974410402e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4189,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14220
|
|
},
|
|
{
|
|
"epoch": 0.5296756062608178,
|
|
"grad_norm": 7.885255695173066,
|
|
"learning_rate": 2.677213896370459e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14230
|
|
},
|
|
{
|
|
"epoch": 0.5300478308611416,
|
|
"grad_norm": 7.531095636542212,
|
|
"learning_rate": 2.6739735191338765e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.9375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 14240
|
|
},
|
|
{
|
|
"epoch": 0.5304200554614654,
|
|
"grad_norm": 6.813751161278367,
|
|
"learning_rate": 2.6707328481714997e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4038,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14250
|
|
},
|
|
{
|
|
"epoch": 0.5307922800617892,
|
|
"grad_norm": 6.911407959797482,
|
|
"learning_rate": 2.6674918889546713e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.378,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14260
|
|
},
|
|
{
|
|
"epoch": 0.5311645046621131,
|
|
"grad_norm": 7.0616203959218335,
|
|
"learning_rate": 2.6642506469552197e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3732,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 14270
|
|
},
|
|
{
|
|
"epoch": 0.531536729262437,
|
|
"grad_norm": 8.01672973609664,
|
|
"learning_rate": 2.6610091276454523e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3983,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14280
|
|
},
|
|
{
|
|
"epoch": 0.5319089538627608,
|
|
"grad_norm": 6.5393312459232416,
|
|
"learning_rate": 2.657767336498142e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3896,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14290
|
|
},
|
|
{
|
|
"epoch": 0.5322811784630846,
|
|
"grad_norm": 6.70091165823984,
|
|
"learning_rate": 2.654525278986523e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3954,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14300
|
|
},
|
|
{
|
|
"epoch": 0.5326534030634085,
|
|
"grad_norm": 6.240817340778308,
|
|
"learning_rate": 2.651282960584279e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.404,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14310
|
|
},
|
|
{
|
|
"epoch": 0.5330256276637323,
|
|
"grad_norm": 6.616733486987542,
|
|
"learning_rate": 2.6480403867655325e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.391,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14320
|
|
},
|
|
{
|
|
"epoch": 0.5333978522640561,
|
|
"grad_norm": 6.618735431027259,
|
|
"learning_rate": 2.644797563004839e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3957,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14330
|
|
},
|
|
{
|
|
"epoch": 0.5337700768643799,
|
|
"grad_norm": 7.049889378384346,
|
|
"learning_rate": 2.641554494777175e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3824,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14340
|
|
},
|
|
{
|
|
"epoch": 0.5341423014647038,
|
|
"grad_norm": 7.62020229970661,
|
|
"learning_rate": 2.6383111875579313e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3985,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14350
|
|
},
|
|
{
|
|
"epoch": 0.5345145260650276,
|
|
"grad_norm": 6.79212427597568,
|
|
"learning_rate": 2.635067646822898e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3939,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 14360
|
|
},
|
|
{
|
|
"epoch": 0.5348867506653515,
|
|
"grad_norm": 7.672002699328915,
|
|
"learning_rate": 2.631823878048266e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3996,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14370
|
|
},
|
|
{
|
|
"epoch": 0.5352589752656753,
|
|
"grad_norm": 6.684112659909468,
|
|
"learning_rate": 2.628579886710605e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4002,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14380
|
|
},
|
|
{
|
|
"epoch": 0.5356311998659992,
|
|
"grad_norm": 8.40090113568921,
|
|
"learning_rate": 2.625335678286864e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14390
|
|
},
|
|
{
|
|
"epoch": 0.536003424466323,
|
|
"grad_norm": 6.852559893392454,
|
|
"learning_rate": 2.622091258254358e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3797,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14400
|
|
},
|
|
{
|
|
"epoch": 0.5363756490666468,
|
|
"grad_norm": 6.249308588478233,
|
|
"learning_rate": 2.618846632090758e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.388,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14410
|
|
},
|
|
{
|
|
"epoch": 0.5367478736669706,
|
|
"grad_norm": 6.484766708073745,
|
|
"learning_rate": 2.6156018052740844e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4115,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14420
|
|
},
|
|
{
|
|
"epoch": 0.5371200982672945,
|
|
"grad_norm": 6.726618591699916,
|
|
"learning_rate": 2.6123567832826955e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14430
|
|
},
|
|
{
|
|
"epoch": 0.5374923228676183,
|
|
"grad_norm": 9.249372537025282,
|
|
"learning_rate": 2.60911157159528e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14440
|
|
},
|
|
{
|
|
"epoch": 0.5378645474679422,
|
|
"grad_norm": 6.582883838294503,
|
|
"learning_rate": 2.6058661756908463e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3722,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14450
|
|
},
|
|
{
|
|
"epoch": 0.538236772068266,
|
|
"grad_norm": 6.51399872210533,
|
|
"learning_rate": 2.6026206010487135e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3984,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14460
|
|
},
|
|
{
|
|
"epoch": 0.5386089966685899,
|
|
"grad_norm": 7.23787882017857,
|
|
"learning_rate": 2.599374853148502e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3852,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14470
|
|
},
|
|
{
|
|
"epoch": 0.5389812212689137,
|
|
"grad_norm": 6.817267543591917,
|
|
"learning_rate": 2.596128937470127e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14480
|
|
},
|
|
{
|
|
"epoch": 0.5393534458692375,
|
|
"grad_norm": 7.576944613734371,
|
|
"learning_rate": 2.592882859493785e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3967,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14490
|
|
},
|
|
{
|
|
"epoch": 0.5397256704695613,
|
|
"grad_norm": 6.420624136931562,
|
|
"learning_rate": 2.5896366246999474e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14500
|
|
},
|
|
{
|
|
"epoch": 0.5400978950698851,
|
|
"grad_norm": 7.265046702394595,
|
|
"learning_rate": 2.586390238569349e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.389,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14510
|
|
},
|
|
{
|
|
"epoch": 0.540470119670209,
|
|
"grad_norm": 7.794165406937612,
|
|
"learning_rate": 2.583143706582983e-07,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14520
|
|
},
|
|
{
|
|
"epoch": 0.5408423442705328,
|
|
"grad_norm": 7.3554022369080645,
|
|
"learning_rate": 2.579897034222084e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3911,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14530
|
|
},
|
|
{
|
|
"epoch": 0.5412145688708567,
|
|
"grad_norm": 8.137724545593255,
|
|
"learning_rate": 2.57665022696813e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3716,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14540
|
|
},
|
|
{
|
|
"epoch": 0.5415867934711805,
|
|
"grad_norm": 8.304468138205376,
|
|
"learning_rate": 2.573403290302819e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3699,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14550
|
|
},
|
|
{
|
|
"epoch": 0.5419590180715044,
|
|
"grad_norm": 6.149804261315394,
|
|
"learning_rate": 2.5701562297080755e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14560
|
|
},
|
|
{
|
|
"epoch": 0.5423312426718282,
|
|
"grad_norm": 8.536409491163447,
|
|
"learning_rate": 2.5669090506660284e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14570
|
|
},
|
|
{
|
|
"epoch": 0.542703467272152,
|
|
"grad_norm": 7.327503659651948,
|
|
"learning_rate": 2.563661758659007e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4084,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14580
|
|
},
|
|
{
|
|
"epoch": 0.5430756918724758,
|
|
"grad_norm": 7.950355932251477,
|
|
"learning_rate": 2.560414359169533e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3812,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14590
|
|
},
|
|
{
|
|
"epoch": 0.5434479164727997,
|
|
"grad_norm": 7.553093424691595,
|
|
"learning_rate": 2.5571668576803083e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3862,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14600
|
|
},
|
|
{
|
|
"epoch": 0.5438201410731235,
|
|
"grad_norm": 6.3744281223844625,
|
|
"learning_rate": 2.553919259674207e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3894,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14610
|
|
},
|
|
{
|
|
"epoch": 0.5441923656734473,
|
|
"grad_norm": 6.510189048279078,
|
|
"learning_rate": 2.5506715706342665e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4102,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14620
|
|
},
|
|
{
|
|
"epoch": 0.5445645902737712,
|
|
"grad_norm": 6.794485247436765,
|
|
"learning_rate": 2.547423796043678e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3732,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14630
|
|
},
|
|
{
|
|
"epoch": 0.5449368148740951,
|
|
"grad_norm": 8.161710654034653,
|
|
"learning_rate": 2.5441759413857764e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14640
|
|
},
|
|
{
|
|
"epoch": 0.5453090394744189,
|
|
"grad_norm": 6.791452889460235,
|
|
"learning_rate": 2.540928012144033e-07,
|
|
"logits/chosen": -3.546875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14650
|
|
},
|
|
{
|
|
"epoch": 0.5456812640747427,
|
|
"grad_norm": 6.538513716874087,
|
|
"learning_rate": 2.537680013802045e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4081,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14660
|
|
},
|
|
{
|
|
"epoch": 0.5460534886750665,
|
|
"grad_norm": 6.866281220645918,
|
|
"learning_rate": 2.534431951843524e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4164,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14670
|
|
},
|
|
{
|
|
"epoch": 0.5464257132753904,
|
|
"grad_norm": 6.254770581474595,
|
|
"learning_rate": 2.5311838317522924e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3846,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14680
|
|
},
|
|
{
|
|
"epoch": 0.5467979378757142,
|
|
"grad_norm": 6.927735484796693,
|
|
"learning_rate": 2.5279356590122674e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3941,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14690
|
|
},
|
|
{
|
|
"epoch": 0.547170162476038,
|
|
"grad_norm": 6.806950987990037,
|
|
"learning_rate": 2.524687439107458e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14700
|
|
},
|
|
{
|
|
"epoch": 0.5475423870763618,
|
|
"grad_norm": 7.0683038647014245,
|
|
"learning_rate": 2.5214391775219506e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4061,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14710
|
|
},
|
|
{
|
|
"epoch": 0.5479146116766858,
|
|
"grad_norm": 6.866027787062228,
|
|
"learning_rate": 2.5181908797399033e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4141,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14720
|
|
},
|
|
{
|
|
"epoch": 0.5482868362770096,
|
|
"grad_norm": 7.195874999298953,
|
|
"learning_rate": 2.5149425512455346e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14730
|
|
},
|
|
{
|
|
"epoch": 0.5486590608773334,
|
|
"grad_norm": 8.144401480284575,
|
|
"learning_rate": 2.5116941975231153e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3961,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14740
|
|
},
|
|
{
|
|
"epoch": 0.5490312854776572,
|
|
"grad_norm": 6.259821556694595,
|
|
"learning_rate": 2.5084458240569595e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14750
|
|
},
|
|
{
|
|
"epoch": 0.549403510077981,
|
|
"grad_norm": 6.450365035605296,
|
|
"learning_rate": 2.5051974363314126e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3668,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 14760
|
|
},
|
|
{
|
|
"epoch": 0.5497757346783049,
|
|
"grad_norm": 7.388565403917839,
|
|
"learning_rate": 2.501949039830846e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.38,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 14770
|
|
},
|
|
{
|
|
"epoch": 0.5501479592786287,
|
|
"grad_norm": 6.4076357880165675,
|
|
"learning_rate": 2.4987006400396445e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -6.84375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14780
|
|
},
|
|
{
|
|
"epoch": 0.5505201838789525,
|
|
"grad_norm": 6.9033393975188515,
|
|
"learning_rate": 2.4954522424422e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.0,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 14790
|
|
},
|
|
{
|
|
"epoch": 0.5508924084792763,
|
|
"grad_norm": 8.353766250014418,
|
|
"learning_rate": 2.492203852522898e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3849,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14800
|
|
},
|
|
{
|
|
"epoch": 0.5512646330796003,
|
|
"grad_norm": 7.541177652844568,
|
|
"learning_rate": 2.4889554757661154e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4107,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 14810
|
|
},
|
|
{
|
|
"epoch": 0.5516368576799241,
|
|
"grad_norm": 6.752543764235121,
|
|
"learning_rate": 2.4857071176562026e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14820
|
|
},
|
|
{
|
|
"epoch": 0.5520090822802479,
|
|
"grad_norm": 6.729869773120602,
|
|
"learning_rate": 2.482458783677481e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3859,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14830
|
|
},
|
|
{
|
|
"epoch": 0.5523813068805717,
|
|
"grad_norm": 6.607998246129173,
|
|
"learning_rate": 2.47921047931423e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3755,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14840
|
|
},
|
|
{
|
|
"epoch": 0.5527535314808956,
|
|
"grad_norm": 7.59842767053571,
|
|
"learning_rate": 2.47596221005068e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 14850
|
|
},
|
|
{
|
|
"epoch": 0.5531257560812194,
|
|
"grad_norm": 6.719783668200659,
|
|
"learning_rate": 2.472713981371002e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 14860
|
|
},
|
|
{
|
|
"epoch": 0.5534979806815432,
|
|
"grad_norm": 6.121126324624705,
|
|
"learning_rate": 2.4694657987592964e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3733,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 14870
|
|
},
|
|
{
|
|
"epoch": 0.553870205281867,
|
|
"grad_norm": 7.47296907241889,
|
|
"learning_rate": 2.4662176676995895e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 14880
|
|
},
|
|
{
|
|
"epoch": 0.554242429882191,
|
|
"grad_norm": 8.539997819863709,
|
|
"learning_rate": 2.4629695936758176e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3858,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 14890
|
|
},
|
|
{
|
|
"epoch": 0.5546146544825148,
|
|
"grad_norm": 8.623043507572621,
|
|
"learning_rate": 2.459721582171822e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4435,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14900
|
|
},
|
|
{
|
|
"epoch": 0.5549868790828386,
|
|
"grad_norm": 6.723867968563131,
|
|
"learning_rate": 2.4564736386713383e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3939,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14910
|
|
},
|
|
{
|
|
"epoch": 0.5553591036831624,
|
|
"grad_norm": 6.588160467546254,
|
|
"learning_rate": 2.453225768657987e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14920
|
|
},
|
|
{
|
|
"epoch": 0.5557313282834863,
|
|
"grad_norm": 6.887541526917924,
|
|
"learning_rate": 2.4499779776152644e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4243,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14930
|
|
},
|
|
{
|
|
"epoch": 0.5561035528838101,
|
|
"grad_norm": 6.744350997889231,
|
|
"learning_rate": 2.446730271026534e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3958,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 14940
|
|
},
|
|
{
|
|
"epoch": 0.5564757774841339,
|
|
"grad_norm": 6.646450878183934,
|
|
"learning_rate": 2.443482654375017e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -880.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 14950
|
|
},
|
|
{
|
|
"epoch": 0.5568480020844577,
|
|
"grad_norm": 5.7469740734323675,
|
|
"learning_rate": 2.440235133143781e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14960
|
|
},
|
|
{
|
|
"epoch": 0.5572202266847815,
|
|
"grad_norm": 6.904225629535094,
|
|
"learning_rate": 2.4369877128157353e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3898,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 14970
|
|
},
|
|
{
|
|
"epoch": 0.5575924512851055,
|
|
"grad_norm": 5.8820128449432945,
|
|
"learning_rate": 2.433740398873616e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 14980
|
|
},
|
|
{
|
|
"epoch": 0.5579646758854293,
|
|
"grad_norm": 9.85087614331916,
|
|
"learning_rate": 2.4304931967999817e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3912,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 14990
|
|
},
|
|
{
|
|
"epoch": 0.5583369004857531,
|
|
"grad_norm": 7.013182637305766,
|
|
"learning_rate": 2.4272461120772e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3861,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15000
|
|
},
|
|
{
|
|
"epoch": 0.558709125086077,
|
|
"grad_norm": 6.444432986204527,
|
|
"learning_rate": 2.4239991501874437e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.417,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15010
|
|
},
|
|
{
|
|
"epoch": 0.5590813496864008,
|
|
"grad_norm": 8.44602991985112,
|
|
"learning_rate": 2.4207523166126737e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15020
|
|
},
|
|
{
|
|
"epoch": 0.5594535742867246,
|
|
"grad_norm": 7.317942146380217,
|
|
"learning_rate": 2.417505616834638e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3903,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15030
|
|
},
|
|
{
|
|
"epoch": 0.5598257988870484,
|
|
"grad_norm": 7.7900011129973485,
|
|
"learning_rate": 2.4142590563348586e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4111,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15040
|
|
},
|
|
{
|
|
"epoch": 0.5601980234873722,
|
|
"grad_norm": 7.786474763272965,
|
|
"learning_rate": 2.411012640594619e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3771,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15050
|
|
},
|
|
{
|
|
"epoch": 0.5605702480876961,
|
|
"grad_norm": 7.798241269104502,
|
|
"learning_rate": 2.4077663750949624e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15060
|
|
},
|
|
{
|
|
"epoch": 0.56094247268802,
|
|
"grad_norm": 7.853052183782864,
|
|
"learning_rate": 2.404520265316675e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15070
|
|
},
|
|
{
|
|
"epoch": 0.5613146972883438,
|
|
"grad_norm": 7.429038497628892,
|
|
"learning_rate": 2.4012743167402823e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4088,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15080
|
|
},
|
|
{
|
|
"epoch": 0.5616869218886676,
|
|
"grad_norm": 7.791803609585541,
|
|
"learning_rate": 2.3980285348460363e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15090
|
|
},
|
|
{
|
|
"epoch": 0.5620591464889915,
|
|
"grad_norm": 7.480886368533306,
|
|
"learning_rate": 2.3947829251139076e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.03125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 15100
|
|
},
|
|
{
|
|
"epoch": 0.5624313710893153,
|
|
"grad_norm": 11.401968261912568,
|
|
"learning_rate": 2.391537493023579e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3892,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15110
|
|
},
|
|
{
|
|
"epoch": 0.5628035956896391,
|
|
"grad_norm": 8.204724117760353,
|
|
"learning_rate": 2.388292244054429e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -872.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.4154,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 15120
|
|
},
|
|
{
|
|
"epoch": 0.5631758202899629,
|
|
"grad_norm": 7.09981766994766,
|
|
"learning_rate": 2.3850471836855297e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 15130
|
|
},
|
|
{
|
|
"epoch": 0.5635480448902868,
|
|
"grad_norm": 7.885238665849159,
|
|
"learning_rate": 2.3818023173956334e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4033,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.75,
|
|
"step": 15140
|
|
},
|
|
{
|
|
"epoch": 0.5639202694906106,
|
|
"grad_norm": 7.648194052957175,
|
|
"learning_rate": 2.3785576506631665e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.40625,
|
|
"logps/chosen": -864.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3811,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -6.96875,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15150
|
|
},
|
|
{
|
|
"epoch": 0.5642924940909345,
|
|
"grad_norm": 8.648532052690246,
|
|
"learning_rate": 2.3753131889662162e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4106,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 15160
|
|
},
|
|
{
|
|
"epoch": 0.5646647186912583,
|
|
"grad_norm": 6.101330987260386,
|
|
"learning_rate": 2.372068937782526e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15170
|
|
},
|
|
{
|
|
"epoch": 0.5650369432915822,
|
|
"grad_norm": 6.164626554390913,
|
|
"learning_rate": 2.368824902589481e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15180
|
|
},
|
|
{
|
|
"epoch": 0.565409167891906,
|
|
"grad_norm": 6.478648485231618,
|
|
"learning_rate": 2.3655810888641044e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3699,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15190
|
|
},
|
|
{
|
|
"epoch": 0.5657813924922298,
|
|
"grad_norm": 6.737756572458076,
|
|
"learning_rate": 2.362337502083045e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4105,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15200
|
|
},
|
|
{
|
|
"epoch": 0.5661536170925536,
|
|
"grad_norm": 6.458253272831201,
|
|
"learning_rate": 2.3590941477225666e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15210
|
|
},
|
|
{
|
|
"epoch": 0.5665258416928775,
|
|
"grad_norm": 7.522363498148898,
|
|
"learning_rate": 2.3558510312585425e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4122,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.2421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 15220
|
|
},
|
|
{
|
|
"epoch": 0.5668980662932013,
|
|
"grad_norm": 7.4661196932671805,
|
|
"learning_rate": 2.3526081581664434e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.4039,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 15230
|
|
},
|
|
{
|
|
"epoch": 0.5672702908935251,
|
|
"grad_norm": 6.89047184777776,
|
|
"learning_rate": 2.3493655339213303e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4087,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 15240
|
|
},
|
|
{
|
|
"epoch": 0.567642515493849,
|
|
"grad_norm": 7.645820280042185,
|
|
"learning_rate": 2.346123163997841e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3961,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15250
|
|
},
|
|
{
|
|
"epoch": 0.5680147400941729,
|
|
"grad_norm": 7.303617426801667,
|
|
"learning_rate": 2.3428810538701893e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3678,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15260
|
|
},
|
|
{
|
|
"epoch": 0.5683869646944967,
|
|
"grad_norm": 8.446406829384378,
|
|
"learning_rate": 2.3396392090121435e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3975,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15270
|
|
},
|
|
{
|
|
"epoch": 0.5687591892948205,
|
|
"grad_norm": 7.123432163957335,
|
|
"learning_rate": 2.3363976348970303e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.409,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15280
|
|
},
|
|
{
|
|
"epoch": 0.5691314138951443,
|
|
"grad_norm": 7.086629386852935,
|
|
"learning_rate": 2.3331563369977163e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.404,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15290
|
|
},
|
|
{
|
|
"epoch": 0.5695036384954681,
|
|
"grad_norm": 7.322084104853638,
|
|
"learning_rate": 2.3299153207866013e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15300
|
|
},
|
|
{
|
|
"epoch": 0.569875863095792,
|
|
"grad_norm": 7.288122182556658,
|
|
"learning_rate": 2.326674591735612e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15310
|
|
},
|
|
{
|
|
"epoch": 0.5702480876961158,
|
|
"grad_norm": 6.420367536684302,
|
|
"learning_rate": 2.3234341553161858e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15320
|
|
},
|
|
{
|
|
"epoch": 0.5706203122964397,
|
|
"grad_norm": 7.126686537484334,
|
|
"learning_rate": 2.3201940169992723e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3893,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15330
|
|
},
|
|
{
|
|
"epoch": 0.5709925368967635,
|
|
"grad_norm": 7.343876485922317,
|
|
"learning_rate": 2.316954182255311e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15340
|
|
},
|
|
{
|
|
"epoch": 0.5713647614970874,
|
|
"grad_norm": 7.433158691008674,
|
|
"learning_rate": 2.3137146565542343e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4081,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15350
|
|
},
|
|
{
|
|
"epoch": 0.5717369860974112,
|
|
"grad_norm": 7.4361078573338775,
|
|
"learning_rate": 2.3104754453654514e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4098,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15360
|
|
},
|
|
{
|
|
"epoch": 0.572109210697735,
|
|
"grad_norm": 7.966604957073583,
|
|
"learning_rate": 2.307236554157838e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3901,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 15370
|
|
},
|
|
{
|
|
"epoch": 0.5724814352980588,
|
|
"grad_norm": 6.688592129151021,
|
|
"learning_rate": 2.3039979883997333e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15380
|
|
},
|
|
{
|
|
"epoch": 0.5728536598983827,
|
|
"grad_norm": 7.591438620217774,
|
|
"learning_rate": 2.3007597535589224e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15390
|
|
},
|
|
{
|
|
"epoch": 0.5732258844987065,
|
|
"grad_norm": 6.849693752912321,
|
|
"learning_rate": 2.297521855102638e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3929,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 15400
|
|
},
|
|
{
|
|
"epoch": 0.5735981090990303,
|
|
"grad_norm": 6.199551085341801,
|
|
"learning_rate": 2.2942842984975383e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15410
|
|
},
|
|
{
|
|
"epoch": 0.5739703336993542,
|
|
"grad_norm": 8.691178447386266,
|
|
"learning_rate": 2.2910470892097102e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3943,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 15420
|
|
},
|
|
{
|
|
"epoch": 0.5743425582996781,
|
|
"grad_norm": 7.232902244404427,
|
|
"learning_rate": 2.287810232704649e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15430
|
|
},
|
|
{
|
|
"epoch": 0.5747147829000019,
|
|
"grad_norm": 7.58606550182947,
|
|
"learning_rate": 2.2845737344472564e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15440
|
|
},
|
|
{
|
|
"epoch": 0.5750870075003257,
|
|
"grad_norm": 6.899994657545038,
|
|
"learning_rate": 2.2813375999018334e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3766,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15450
|
|
},
|
|
{
|
|
"epoch": 0.5754592321006495,
|
|
"grad_norm": 7.713133621602367,
|
|
"learning_rate": 2.2781018345320585e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3915,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15460
|
|
},
|
|
{
|
|
"epoch": 0.5758314567009734,
|
|
"grad_norm": 8.48425661744594,
|
|
"learning_rate": 2.2748664438009953e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3853,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 15470
|
|
},
|
|
{
|
|
"epoch": 0.5762036813012972,
|
|
"grad_norm": 7.9707739483281035,
|
|
"learning_rate": 2.2716314331710684e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.383,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15480
|
|
},
|
|
{
|
|
"epoch": 0.576575905901621,
|
|
"grad_norm": 8.320472488321666,
|
|
"learning_rate": 2.268396808104066e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15490
|
|
},
|
|
{
|
|
"epoch": 0.5769481305019448,
|
|
"grad_norm": 6.512408404644271,
|
|
"learning_rate": 2.2651625740611206e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 15500
|
|
},
|
|
{
|
|
"epoch": 0.5773203551022688,
|
|
"grad_norm": 6.239999758950461,
|
|
"learning_rate": 2.2619287365027075e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3898,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15510
|
|
},
|
|
{
|
|
"epoch": 0.5776925797025926,
|
|
"grad_norm": 7.476903872711185,
|
|
"learning_rate": 2.2586953008886314e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3841,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15520
|
|
},
|
|
{
|
|
"epoch": 0.5780648043029164,
|
|
"grad_norm": 6.412930684722046,
|
|
"learning_rate": 2.2554622726780186e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3701,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15530
|
|
},
|
|
{
|
|
"epoch": 0.5784370289032402,
|
|
"grad_norm": 7.342840011702307,
|
|
"learning_rate": 2.25222965732931e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4158,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15540
|
|
},
|
|
{
|
|
"epoch": 0.578809253503564,
|
|
"grad_norm": 6.903254264492447,
|
|
"learning_rate": 2.2489974603002437e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4258,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15550
|
|
},
|
|
{
|
|
"epoch": 0.5791814781038879,
|
|
"grad_norm": 6.516327693323662,
|
|
"learning_rate": 2.2457656870478587e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3929,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15560
|
|
},
|
|
{
|
|
"epoch": 0.5795537027042117,
|
|
"grad_norm": 6.933471590009466,
|
|
"learning_rate": 2.2425343430284716e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3944,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15570
|
|
},
|
|
{
|
|
"epoch": 0.5799259273045355,
|
|
"grad_norm": 7.043923500085154,
|
|
"learning_rate": 2.239303433697679e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15580
|
|
},
|
|
{
|
|
"epoch": 0.5802981519048593,
|
|
"grad_norm": 8.62504327687977,
|
|
"learning_rate": 2.2360729645103418e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3942,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15590
|
|
},
|
|
{
|
|
"epoch": 0.5806703765051833,
|
|
"grad_norm": 7.8375934125781255,
|
|
"learning_rate": 2.232842940920579e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3927,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 15600
|
|
},
|
|
{
|
|
"epoch": 0.5810426011055071,
|
|
"grad_norm": 8.238458995209937,
|
|
"learning_rate": 2.229613368381755e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3986,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15610
|
|
},
|
|
{
|
|
"epoch": 0.5814148257058309,
|
|
"grad_norm": 8.184100544423176,
|
|
"learning_rate": 2.2263842523464744e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3859,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 15620
|
|
},
|
|
{
|
|
"epoch": 0.5817870503061547,
|
|
"grad_norm": 7.4289363426874315,
|
|
"learning_rate": 2.2231555982665728e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15630
|
|
},
|
|
{
|
|
"epoch": 0.5821592749064786,
|
|
"grad_norm": 8.990519330811633,
|
|
"learning_rate": 2.2199274115931012e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3853,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15640
|
|
},
|
|
{
|
|
"epoch": 0.5825314995068024,
|
|
"grad_norm": 7.089366672162368,
|
|
"learning_rate": 2.216699697776326e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3762,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15650
|
|
},
|
|
{
|
|
"epoch": 0.5829037241071262,
|
|
"grad_norm": 6.241261251342792,
|
|
"learning_rate": 2.2134724622657114e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 15660
|
|
},
|
|
{
|
|
"epoch": 0.58327594870745,
|
|
"grad_norm": 7.968299568981501,
|
|
"learning_rate": 2.2102457105099177e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3935,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15670
|
|
},
|
|
{
|
|
"epoch": 0.583648173307774,
|
|
"grad_norm": 7.948511461048663,
|
|
"learning_rate": 2.2070194479567858e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4047,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15680
|
|
},
|
|
{
|
|
"epoch": 0.5840203979080978,
|
|
"grad_norm": 7.43424918957294,
|
|
"learning_rate": 2.2037936800533319e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.381,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15690
|
|
},
|
|
{
|
|
"epoch": 0.5843926225084216,
|
|
"grad_norm": 8.129342771397607,
|
|
"learning_rate": 2.2005684122457374e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15700
|
|
},
|
|
{
|
|
"epoch": 0.5847648471087454,
|
|
"grad_norm": 6.831357443127663,
|
|
"learning_rate": 2.1973436499793377e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3918,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15710
|
|
},
|
|
{
|
|
"epoch": 0.5851370717090693,
|
|
"grad_norm": 7.5769140061916636,
|
|
"learning_rate": 2.194119398698617e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15720
|
|
},
|
|
{
|
|
"epoch": 0.5855092963093931,
|
|
"grad_norm": 7.143948979709037,
|
|
"learning_rate": 2.190895663847194e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3779,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15730
|
|
},
|
|
{
|
|
"epoch": 0.5858815209097169,
|
|
"grad_norm": 6.37709548249504,
|
|
"learning_rate": 2.1876724508678184e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3834,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15740
|
|
},
|
|
{
|
|
"epoch": 0.5862537455100407,
|
|
"grad_norm": 6.581300666651551,
|
|
"learning_rate": 2.1844497652023563e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3937,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15750
|
|
},
|
|
{
|
|
"epoch": 0.5866259701103645,
|
|
"grad_norm": 5.734998112720363,
|
|
"learning_rate": 2.181227612291786e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3751,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15760
|
|
},
|
|
{
|
|
"epoch": 0.5869981947106885,
|
|
"grad_norm": 6.535808641370942,
|
|
"learning_rate": 2.1780059975761833e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15770
|
|
},
|
|
{
|
|
"epoch": 0.5873704193110123,
|
|
"grad_norm": 6.396799240741715,
|
|
"learning_rate": 2.1747849264947177e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3774,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.0625,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15780
|
|
},
|
|
{
|
|
"epoch": 0.5877426439113361,
|
|
"grad_norm": 6.710808676868262,
|
|
"learning_rate": 2.1715644044856413e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3903,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15790
|
|
},
|
|
{
|
|
"epoch": 0.58811486851166,
|
|
"grad_norm": 8.033202311280913,
|
|
"learning_rate": 2.1683444369862763e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15800
|
|
},
|
|
{
|
|
"epoch": 0.5884870931119838,
|
|
"grad_norm": 7.582213474536643,
|
|
"learning_rate": 2.165125029433012e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15810
|
|
},
|
|
{
|
|
"epoch": 0.5888593177123076,
|
|
"grad_norm": 6.823963875901322,
|
|
"learning_rate": 2.1619061872612892e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4103,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15820
|
|
},
|
|
{
|
|
"epoch": 0.5892315423126314,
|
|
"grad_norm": 6.814558152122144,
|
|
"learning_rate": 2.158687915905597e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3798,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15830
|
|
},
|
|
{
|
|
"epoch": 0.5896037669129552,
|
|
"grad_norm": 6.105379495009104,
|
|
"learning_rate": 2.155470220799459e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3958,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15840
|
|
},
|
|
{
|
|
"epoch": 0.5899759915132791,
|
|
"grad_norm": 6.657287329471462,
|
|
"learning_rate": 2.152253107375427e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3925,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15850
|
|
},
|
|
{
|
|
"epoch": 0.590348216113603,
|
|
"grad_norm": 7.679202933407388,
|
|
"learning_rate": 2.1490365810650686e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 15860
|
|
},
|
|
{
|
|
"epoch": 0.5907204407139268,
|
|
"grad_norm": 8.639484529782708,
|
|
"learning_rate": 2.1458206472989626e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3714,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 15870
|
|
},
|
|
{
|
|
"epoch": 0.5910926653142506,
|
|
"grad_norm": 7.599863240783811,
|
|
"learning_rate": 2.1426053115066875e-07,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15880
|
|
},
|
|
{
|
|
"epoch": 0.5914648899145745,
|
|
"grad_norm": 6.383338257877059,
|
|
"learning_rate": 2.1393905791168093e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -876.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3743,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 15890
|
|
},
|
|
{
|
|
"epoch": 0.5918371145148983,
|
|
"grad_norm": 7.92112382568732,
|
|
"learning_rate": 2.136176455556879e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3793,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15900
|
|
},
|
|
{
|
|
"epoch": 0.5922093391152221,
|
|
"grad_norm": 6.957102367208679,
|
|
"learning_rate": 2.132962946253416e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3793,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15910
|
|
},
|
|
{
|
|
"epoch": 0.5925815637155459,
|
|
"grad_norm": 8.051681457929154,
|
|
"learning_rate": 2.129750056631906e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15920
|
|
},
|
|
{
|
|
"epoch": 0.5929537883158698,
|
|
"grad_norm": 8.458061669162245,
|
|
"learning_rate": 2.1265377921167855e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3845,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 15930
|
|
},
|
|
{
|
|
"epoch": 0.5933260129161936,
|
|
"grad_norm": 7.491726001458984,
|
|
"learning_rate": 2.1233261581314385e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3888,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 15940
|
|
},
|
|
{
|
|
"epoch": 0.5936982375165175,
|
|
"grad_norm": 7.753580401594826,
|
|
"learning_rate": 2.1201151600981814e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 15950
|
|
},
|
|
{
|
|
"epoch": 0.5940704621168413,
|
|
"grad_norm": 7.3447749832639415,
|
|
"learning_rate": 2.1169048034382598e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.388,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 15960
|
|
},
|
|
{
|
|
"epoch": 0.5944426867171652,
|
|
"grad_norm": 6.134838918206232,
|
|
"learning_rate": 2.1136950935718348e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.421,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 15970
|
|
},
|
|
{
|
|
"epoch": 0.594814911317489,
|
|
"grad_norm": 7.794041338498578,
|
|
"learning_rate": 2.1104860359179756e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3789,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 15980
|
|
},
|
|
{
|
|
"epoch": 0.5951871359178128,
|
|
"grad_norm": 8.802551494776141,
|
|
"learning_rate": 2.1072776358946507e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3565,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 15990
|
|
},
|
|
{
|
|
"epoch": 0.5955593605181366,
|
|
"grad_norm": 9.05082905569019,
|
|
"learning_rate": 2.1040698989187175e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.362,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 16000
|
|
},
|
|
{
|
|
"epoch": 0.5959315851184604,
|
|
"grad_norm": 7.628745379503463,
|
|
"learning_rate": 2.1008628304059155e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.389,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16010
|
|
},
|
|
{
|
|
"epoch": 0.5963038097187843,
|
|
"grad_norm": 7.781906658850455,
|
|
"learning_rate": 2.0976564357708538e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3975,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 16020
|
|
},
|
|
{
|
|
"epoch": 0.5966760343191081,
|
|
"grad_norm": 6.442990219172427,
|
|
"learning_rate": 2.0944507204270044e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16030
|
|
},
|
|
{
|
|
"epoch": 0.597048258919432,
|
|
"grad_norm": 6.3353866789721955,
|
|
"learning_rate": 2.0912456897866942e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4063,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16040
|
|
},
|
|
{
|
|
"epoch": 0.5974204835197559,
|
|
"grad_norm": 6.471483700991728,
|
|
"learning_rate": 2.0880413492610904e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 16050
|
|
},
|
|
{
|
|
"epoch": 0.5977927081200797,
|
|
"grad_norm": 7.190245141355277,
|
|
"learning_rate": 2.0848377042601992e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4029,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 16060
|
|
},
|
|
{
|
|
"epoch": 0.5981649327204035,
|
|
"grad_norm": 6.807449670556438,
|
|
"learning_rate": 2.081634760192849e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4129,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16070
|
|
},
|
|
{
|
|
"epoch": 0.5985371573207273,
|
|
"grad_norm": 7.983332072321888,
|
|
"learning_rate": 2.078432522466687e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3844,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16080
|
|
},
|
|
{
|
|
"epoch": 0.5989093819210511,
|
|
"grad_norm": 7.924151344761925,
|
|
"learning_rate": 2.0752309964881664e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4101,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16090
|
|
},
|
|
{
|
|
"epoch": 0.599281606521375,
|
|
"grad_norm": 7.569869963047868,
|
|
"learning_rate": 2.0720301876625416e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.411,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16100
|
|
},
|
|
{
|
|
"epoch": 0.5996538311216988,
|
|
"grad_norm": 7.901064972967728,
|
|
"learning_rate": 2.0688301013938504e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3834,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16110
|
|
},
|
|
{
|
|
"epoch": 0.6000260557220227,
|
|
"grad_norm": 6.45733508716788,
|
|
"learning_rate": 2.065630743084917e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16120
|
|
},
|
|
{
|
|
"epoch": 0.6003982803223465,
|
|
"grad_norm": 7.426896298134275,
|
|
"learning_rate": 2.062432118137334e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 16130
|
|
},
|
|
{
|
|
"epoch": 0.6007705049226704,
|
|
"grad_norm": 7.059616800852437,
|
|
"learning_rate": 2.0592342319514547e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3839,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16140
|
|
},
|
|
{
|
|
"epoch": 0.6011427295229942,
|
|
"grad_norm": 7.745742966520787,
|
|
"learning_rate": 2.0560370899263867e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16150
|
|
},
|
|
{
|
|
"epoch": 0.601514954123318,
|
|
"grad_norm": 7.133218282085067,
|
|
"learning_rate": 2.0528406974599808e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16160
|
|
},
|
|
{
|
|
"epoch": 0.6018871787236418,
|
|
"grad_norm": 7.482295640462812,
|
|
"learning_rate": 2.0496450599488224e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3803,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16170
|
|
},
|
|
{
|
|
"epoch": 0.6022594033239657,
|
|
"grad_norm": 8.867126684542379,
|
|
"learning_rate": 2.0464501827882202e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3918,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 16180
|
|
},
|
|
{
|
|
"epoch": 0.6026316279242895,
|
|
"grad_norm": 7.21391132753523,
|
|
"learning_rate": 2.0432560713722043e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16190
|
|
},
|
|
{
|
|
"epoch": 0.6030038525246133,
|
|
"grad_norm": 7.27989588298991,
|
|
"learning_rate": 2.040062731093505e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3915,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16200
|
|
},
|
|
{
|
|
"epoch": 0.6033760771249372,
|
|
"grad_norm": 8.902539763193891,
|
|
"learning_rate": 2.0368701673435567e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3687,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 16210
|
|
},
|
|
{
|
|
"epoch": 0.6037483017252611,
|
|
"grad_norm": 6.65900347493812,
|
|
"learning_rate": 2.0336783855124806e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3829,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16220
|
|
},
|
|
{
|
|
"epoch": 0.6041205263255849,
|
|
"grad_norm": 8.329875560565936,
|
|
"learning_rate": 2.0304873909890763e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3951,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16230
|
|
},
|
|
{
|
|
"epoch": 0.6044927509259087,
|
|
"grad_norm": 7.604049926199696,
|
|
"learning_rate": 2.027297189160817e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3927,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 16240
|
|
},
|
|
{
|
|
"epoch": 0.6048649755262325,
|
|
"grad_norm": 6.095888601514059,
|
|
"learning_rate": 2.0241077854138337e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16250
|
|
},
|
|
{
|
|
"epoch": 0.6052372001265564,
|
|
"grad_norm": 7.956051067632771,
|
|
"learning_rate": 2.0209191851329149e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16260
|
|
},
|
|
{
|
|
"epoch": 0.6056094247268802,
|
|
"grad_norm": 5.76669626458618,
|
|
"learning_rate": 2.0177313937014873e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3851,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16270
|
|
},
|
|
{
|
|
"epoch": 0.605981649327204,
|
|
"grad_norm": 7.2141933056882195,
|
|
"learning_rate": 2.0145444165016164e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3985,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 16280
|
|
},
|
|
{
|
|
"epoch": 0.6063538739275278,
|
|
"grad_norm": 5.698726018133921,
|
|
"learning_rate": 2.0113582589139917e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3891,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16290
|
|
},
|
|
{
|
|
"epoch": 0.6067260985278518,
|
|
"grad_norm": 8.211986668738497,
|
|
"learning_rate": 2.0081729263179165e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.371,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 16300
|
|
},
|
|
{
|
|
"epoch": 0.6070983231281756,
|
|
"grad_norm": 7.658026739862167,
|
|
"learning_rate": 2.0049884240913056e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16310
|
|
},
|
|
{
|
|
"epoch": 0.6074705477284994,
|
|
"grad_norm": 6.766536645408155,
|
|
"learning_rate": 2.0018047576106666e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3791,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16320
|
|
},
|
|
{
|
|
"epoch": 0.6078427723288232,
|
|
"grad_norm": 9.235816293554157,
|
|
"learning_rate": 1.998621932251102e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.371,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16330
|
|
},
|
|
{
|
|
"epoch": 0.608214996929147,
|
|
"grad_norm": 6.9251806952866835,
|
|
"learning_rate": 1.9954399533862883e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4067,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16340
|
|
},
|
|
{
|
|
"epoch": 0.6085872215294709,
|
|
"grad_norm": 7.946145070185554,
|
|
"learning_rate": 1.9922588263884785e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16350
|
|
},
|
|
{
|
|
"epoch": 0.6089594461297947,
|
|
"grad_norm": 6.558157803053308,
|
|
"learning_rate": 1.9890785566284817e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3703,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16360
|
|
},
|
|
{
|
|
"epoch": 0.6093316707301185,
|
|
"grad_norm": 9.422626945248469,
|
|
"learning_rate": 1.9858991494756635e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16370
|
|
},
|
|
{
|
|
"epoch": 0.6097038953304423,
|
|
"grad_norm": 7.334966152468533,
|
|
"learning_rate": 1.9827206102979343e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16380
|
|
},
|
|
{
|
|
"epoch": 0.6100761199307663,
|
|
"grad_norm": 6.5555286780333315,
|
|
"learning_rate": 1.9795429444617334e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4169,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 16390
|
|
},
|
|
{
|
|
"epoch": 0.6104483445310901,
|
|
"grad_norm": 6.933459933150522,
|
|
"learning_rate": 1.9763661573320322e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4057,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16400
|
|
},
|
|
{
|
|
"epoch": 0.6108205691314139,
|
|
"grad_norm": 7.20705889382901,
|
|
"learning_rate": 1.9731902542723128e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.424,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 16410
|
|
},
|
|
{
|
|
"epoch": 0.6111927937317377,
|
|
"grad_norm": 7.188388925359692,
|
|
"learning_rate": 1.9700152406445704e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16420
|
|
},
|
|
{
|
|
"epoch": 0.6115650183320616,
|
|
"grad_norm": 10.867554833402279,
|
|
"learning_rate": 1.9668411218092928e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 16430
|
|
},
|
|
{
|
|
"epoch": 0.6119372429323854,
|
|
"grad_norm": 7.057677973046077,
|
|
"learning_rate": 1.963667903125461e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16440
|
|
},
|
|
{
|
|
"epoch": 0.6123094675327092,
|
|
"grad_norm": 7.524127017639635,
|
|
"learning_rate": 1.9604955899505346e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16450
|
|
},
|
|
{
|
|
"epoch": 0.612681692133033,
|
|
"grad_norm": 6.4762868704095045,
|
|
"learning_rate": 1.9573241876404445e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16460
|
|
},
|
|
{
|
|
"epoch": 0.613053916733357,
|
|
"grad_norm": 6.870000649506464,
|
|
"learning_rate": 1.9541537015495864e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3806,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16470
|
|
},
|
|
{
|
|
"epoch": 0.6134261413336808,
|
|
"grad_norm": 6.8587017498438785,
|
|
"learning_rate": 1.9509841370308034e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4045,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16480
|
|
},
|
|
{
|
|
"epoch": 0.6137983659340046,
|
|
"grad_norm": 7.503434315330606,
|
|
"learning_rate": 1.9478154994353897e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16490
|
|
},
|
|
{
|
|
"epoch": 0.6141705905343284,
|
|
"grad_norm": 9.067671508683446,
|
|
"learning_rate": 1.9446477941130677e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 16500
|
|
},
|
|
{
|
|
"epoch": 0.6145428151346523,
|
|
"grad_norm": 8.537143934613066,
|
|
"learning_rate": 1.9414810264119907e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3946,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16510
|
|
},
|
|
{
|
|
"epoch": 0.6149150397349761,
|
|
"grad_norm": 7.549038187051423,
|
|
"learning_rate": 1.9383152016787266e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3812,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16520
|
|
},
|
|
{
|
|
"epoch": 0.6152872643352999,
|
|
"grad_norm": 7.894463199968102,
|
|
"learning_rate": 1.935150325258253e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3808,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16530
|
|
},
|
|
{
|
|
"epoch": 0.6156594889356237,
|
|
"grad_norm": 7.680931906675422,
|
|
"learning_rate": 1.931986402493944e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3911,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16540
|
|
},
|
|
{
|
|
"epoch": 0.6160317135359475,
|
|
"grad_norm": 7.079423194577833,
|
|
"learning_rate": 1.9288234387275656e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 16550
|
|
},
|
|
{
|
|
"epoch": 0.6164039381362715,
|
|
"grad_norm": 7.704503467287907,
|
|
"learning_rate": 1.9256614392992647e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3845,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16560
|
|
},
|
|
{
|
|
"epoch": 0.6167761627365953,
|
|
"grad_norm": 7.621645594695648,
|
|
"learning_rate": 1.9225004095475587e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3942,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 16570
|
|
},
|
|
{
|
|
"epoch": 0.6171483873369191,
|
|
"grad_norm": 7.27998522062956,
|
|
"learning_rate": 1.9193403548093294e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4094,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.75,
|
|
"step": 16580
|
|
},
|
|
{
|
|
"epoch": 0.617520611937243,
|
|
"grad_norm": 6.521180949257311,
|
|
"learning_rate": 1.9161812804198107e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16590
|
|
},
|
|
{
|
|
"epoch": 0.6178928365375668,
|
|
"grad_norm": 7.337599225519535,
|
|
"learning_rate": 1.9130231917125833e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.09375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 16600
|
|
},
|
|
{
|
|
"epoch": 0.6182650611378906,
|
|
"grad_norm": 8.436069035321404,
|
|
"learning_rate": 1.9098660940195618e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3926,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 16610
|
|
},
|
|
{
|
|
"epoch": 0.6186372857382144,
|
|
"grad_norm": 7.298667972443575,
|
|
"learning_rate": 1.906709992670989e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4171,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16620
|
|
},
|
|
{
|
|
"epoch": 0.6190095103385382,
|
|
"grad_norm": 7.629604814436547,
|
|
"learning_rate": 1.9035548929954257e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16630
|
|
},
|
|
{
|
|
"epoch": 0.6193817349388621,
|
|
"grad_norm": 8.156859170492046,
|
|
"learning_rate": 1.9004008003197398e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3974,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16640
|
|
},
|
|
{
|
|
"epoch": 0.619753959539186,
|
|
"grad_norm": 5.729886266118054,
|
|
"learning_rate": 1.8972477199691014e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -868.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.3739,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -6.90625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 16650
|
|
},
|
|
{
|
|
"epoch": 0.6201261841395098,
|
|
"grad_norm": 6.85626166710456,
|
|
"learning_rate": 1.8940956572669692e-07,
|
|
"logits/chosen": -3.578125,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4072,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16660
|
|
},
|
|
{
|
|
"epoch": 0.6204984087398336,
|
|
"grad_norm": 6.387160876883699,
|
|
"learning_rate": 1.8909446175350856e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3808,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 16670
|
|
},
|
|
{
|
|
"epoch": 0.6208706333401575,
|
|
"grad_norm": 7.649700126342195,
|
|
"learning_rate": 1.887794606093465e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3791,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16680
|
|
},
|
|
{
|
|
"epoch": 0.6212428579404813,
|
|
"grad_norm": 7.26401392292883,
|
|
"learning_rate": 1.8846456282603856e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.385,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16690
|
|
},
|
|
{
|
|
"epoch": 0.6216150825408051,
|
|
"grad_norm": 7.067286641351038,
|
|
"learning_rate": 1.8814976893523804e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16700
|
|
},
|
|
{
|
|
"epoch": 0.6219873071411289,
|
|
"grad_norm": 7.713221539137964,
|
|
"learning_rate": 1.8783507946842291e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3862,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16710
|
|
},
|
|
{
|
|
"epoch": 0.6223595317414528,
|
|
"grad_norm": 7.483835766456431,
|
|
"learning_rate": 1.8752049495689483e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4096,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 16720
|
|
},
|
|
{
|
|
"epoch": 0.6227317563417766,
|
|
"grad_norm": 6.9204759207978865,
|
|
"learning_rate": 1.872060159317782e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3602,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 16730
|
|
},
|
|
{
|
|
"epoch": 0.6231039809421005,
|
|
"grad_norm": 7.1054166402638215,
|
|
"learning_rate": 1.8689164292401932e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3958,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16740
|
|
},
|
|
{
|
|
"epoch": 0.6234762055424243,
|
|
"grad_norm": 9.241952282153285,
|
|
"learning_rate": 1.865773764643855e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4116,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 16750
|
|
},
|
|
{
|
|
"epoch": 0.6238484301427482,
|
|
"grad_norm": 6.818520128943154,
|
|
"learning_rate": 1.8626321708346432e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16760
|
|
},
|
|
{
|
|
"epoch": 0.624220654743072,
|
|
"grad_norm": 7.10993491365885,
|
|
"learning_rate": 1.8594916531166226e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4057,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16770
|
|
},
|
|
{
|
|
"epoch": 0.6245928793433958,
|
|
"grad_norm": 8.617784845013855,
|
|
"learning_rate": 1.8563522167920445e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3755,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 16780
|
|
},
|
|
{
|
|
"epoch": 0.6249651039437196,
|
|
"grad_norm": 8.251507601145008,
|
|
"learning_rate": 1.853213867161332e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 16790
|
|
},
|
|
{
|
|
"epoch": 0.6253373285440434,
|
|
"grad_norm": 7.702940693853803,
|
|
"learning_rate": 1.8500766095230746e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16800
|
|
},
|
|
{
|
|
"epoch": 0.6257095531443673,
|
|
"grad_norm": 6.860310125449067,
|
|
"learning_rate": 1.8469404491740193e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3941,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16810
|
|
},
|
|
{
|
|
"epoch": 0.6260817777446911,
|
|
"grad_norm": 5.954739533579795,
|
|
"learning_rate": 1.8438053914090574e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3939,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16820
|
|
},
|
|
{
|
|
"epoch": 0.626454002345015,
|
|
"grad_norm": 7.053159514642265,
|
|
"learning_rate": 1.8406714415212216e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 16830
|
|
},
|
|
{
|
|
"epoch": 0.6268262269453388,
|
|
"grad_norm": 7.078896928700415,
|
|
"learning_rate": 1.8375386048016722e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16840
|
|
},
|
|
{
|
|
"epoch": 0.6271984515456627,
|
|
"grad_norm": 6.603768256895527,
|
|
"learning_rate": 1.8344068865396922e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3669,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16850
|
|
},
|
|
{
|
|
"epoch": 0.6275706761459865,
|
|
"grad_norm": 7.416092250417769,
|
|
"learning_rate": 1.8312762920226731e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3911,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16860
|
|
},
|
|
{
|
|
"epoch": 0.6279429007463103,
|
|
"grad_norm": 7.714260708297878,
|
|
"learning_rate": 1.8281468265361125e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4092,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 16870
|
|
},
|
|
{
|
|
"epoch": 0.6283151253466341,
|
|
"grad_norm": 8.390090501476644,
|
|
"learning_rate": 1.8250184953636e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16880
|
|
},
|
|
{
|
|
"epoch": 0.628687349946958,
|
|
"grad_norm": 7.621181684559321,
|
|
"learning_rate": 1.8218913037868102e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16890
|
|
},
|
|
{
|
|
"epoch": 0.6290595745472818,
|
|
"grad_norm": 8.344547183952535,
|
|
"learning_rate": 1.8187652570854945e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3817,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 16900
|
|
},
|
|
{
|
|
"epoch": 0.6294317991476057,
|
|
"grad_norm": 7.236022908879116,
|
|
"learning_rate": 1.81564036053747e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4019,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 16910
|
|
},
|
|
{
|
|
"epoch": 0.6298040237479295,
|
|
"grad_norm": 10.993077618017269,
|
|
"learning_rate": 1.812516619418613e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.3974,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 16920
|
|
},
|
|
{
|
|
"epoch": 0.6301762483482534,
|
|
"grad_norm": 6.687269662147331,
|
|
"learning_rate": 1.8093940390028483e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 16930
|
|
},
|
|
{
|
|
"epoch": 0.6305484729485772,
|
|
"grad_norm": 6.651364412921914,
|
|
"learning_rate": 1.806272624562143e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3819,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 16940
|
|
},
|
|
{
|
|
"epoch": 0.630920697548901,
|
|
"grad_norm": 6.774722275594347,
|
|
"learning_rate": 1.8031523813664923e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3672,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16950
|
|
},
|
|
{
|
|
"epoch": 0.6312929221492248,
|
|
"grad_norm": 7.487929516424714,
|
|
"learning_rate": 1.800033314683917e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 16960
|
|
},
|
|
{
|
|
"epoch": 0.6316651467495487,
|
|
"grad_norm": 9.076295749632079,
|
|
"learning_rate": 1.7969154297804507e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4044,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 16970
|
|
},
|
|
{
|
|
"epoch": 0.6320373713498725,
|
|
"grad_norm": 8.612228676413666,
|
|
"learning_rate": 1.7937987319201297e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3798,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 16980
|
|
},
|
|
{
|
|
"epoch": 0.6324095959501963,
|
|
"grad_norm": 6.967363509589258,
|
|
"learning_rate": 1.79068322636499e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3979,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 16990
|
|
},
|
|
{
|
|
"epoch": 0.6327818205505202,
|
|
"grad_norm": 7.196791640655572,
|
|
"learning_rate": 1.7875689183750504e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 17000
|
|
},
|
|
{
|
|
"epoch": 0.6331540451508441,
|
|
"grad_norm": 7.8293420888350695,
|
|
"learning_rate": 1.7844558132083117e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.3976,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 17010
|
|
},
|
|
{
|
|
"epoch": 0.6335262697511679,
|
|
"grad_norm": 7.254639101252117,
|
|
"learning_rate": 1.7813439161207412e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3849,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17020
|
|
},
|
|
{
|
|
"epoch": 0.6338984943514917,
|
|
"grad_norm": 6.9340569235184635,
|
|
"learning_rate": 1.778233232366268e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17030
|
|
},
|
|
{
|
|
"epoch": 0.6342707189518155,
|
|
"grad_norm": 6.705317377655664,
|
|
"learning_rate": 1.7751237671967707e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.378,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17040
|
|
},
|
|
{
|
|
"epoch": 0.6346429435521393,
|
|
"grad_norm": 8.397254264119065,
|
|
"learning_rate": 1.772015525862073e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.875,
|
|
"step": 17050
|
|
},
|
|
{
|
|
"epoch": 0.6350151681524632,
|
|
"grad_norm": 8.299730798555709,
|
|
"learning_rate": 1.7689085136099325e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3613,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17060
|
|
},
|
|
{
|
|
"epoch": 0.635387392752787,
|
|
"grad_norm": 7.431229722860441,
|
|
"learning_rate": 1.7658027356860284e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3867,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17070
|
|
},
|
|
{
|
|
"epoch": 0.6357596173531108,
|
|
"grad_norm": 9.986094553338852,
|
|
"learning_rate": 1.7626981973339597e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4093,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17080
|
|
},
|
|
{
|
|
"epoch": 0.6361318419534348,
|
|
"grad_norm": 6.858702537778318,
|
|
"learning_rate": 1.7595949037952298e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3815,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17090
|
|
},
|
|
{
|
|
"epoch": 0.6365040665537586,
|
|
"grad_norm": 7.916443270507768,
|
|
"learning_rate": 1.7564928603092432e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17100
|
|
},
|
|
{
|
|
"epoch": 0.6368762911540824,
|
|
"grad_norm": 6.923588495858665,
|
|
"learning_rate": 1.7533920721132894e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3699,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17110
|
|
},
|
|
{
|
|
"epoch": 0.6372485157544062,
|
|
"grad_norm": 7.25606825655809,
|
|
"learning_rate": 1.7502925444425448e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3944,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17120
|
|
},
|
|
{
|
|
"epoch": 0.63762074035473,
|
|
"grad_norm": 7.656960751011358,
|
|
"learning_rate": 1.7471942825300512e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3898,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17130
|
|
},
|
|
{
|
|
"epoch": 0.6379929649550539,
|
|
"grad_norm": 7.871333394009538,
|
|
"learning_rate": 1.744097291606718e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3622,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17140
|
|
},
|
|
{
|
|
"epoch": 0.6383651895553777,
|
|
"grad_norm": 8.31614597006072,
|
|
"learning_rate": 1.741001576901308e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3466,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17150
|
|
},
|
|
{
|
|
"epoch": 0.6387374141557015,
|
|
"grad_norm": 7.2106490122825475,
|
|
"learning_rate": 1.7379071436404268e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17160
|
|
},
|
|
{
|
|
"epoch": 0.6391096387560253,
|
|
"grad_norm": 10.768479312293898,
|
|
"learning_rate": 1.73481399704852e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.411,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17170
|
|
},
|
|
{
|
|
"epoch": 0.6394818633563493,
|
|
"grad_norm": 7.675620042920287,
|
|
"learning_rate": 1.7317221423478556e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3973,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17180
|
|
},
|
|
{
|
|
"epoch": 0.6398540879566731,
|
|
"grad_norm": 9.229976917407015,
|
|
"learning_rate": 1.728631584758528e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17190
|
|
},
|
|
{
|
|
"epoch": 0.6402263125569969,
|
|
"grad_norm": 8.732968087554458,
|
|
"learning_rate": 1.7255423294984342e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3851,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17200
|
|
},
|
|
{
|
|
"epoch": 0.6405985371573207,
|
|
"grad_norm": 6.643487652485536,
|
|
"learning_rate": 1.7224543817832777e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3936,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17210
|
|
},
|
|
{
|
|
"epoch": 0.6409707617576446,
|
|
"grad_norm": 7.792267703637605,
|
|
"learning_rate": 1.719367746826553e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.379,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17220
|
|
},
|
|
{
|
|
"epoch": 0.6413429863579684,
|
|
"grad_norm": 7.297208848923311,
|
|
"learning_rate": 1.716282429839535e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3955,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17230
|
|
},
|
|
{
|
|
"epoch": 0.6417152109582922,
|
|
"grad_norm": 6.72489782152116,
|
|
"learning_rate": 1.7131984360312795e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3717,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17240
|
|
},
|
|
{
|
|
"epoch": 0.642087435558616,
|
|
"grad_norm": 6.726838787679959,
|
|
"learning_rate": 1.7101157706086016e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3747,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17250
|
|
},
|
|
{
|
|
"epoch": 0.6424596601589398,
|
|
"grad_norm": 7.15010579362331,
|
|
"learning_rate": 1.70703443877608e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4167,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17260
|
|
},
|
|
{
|
|
"epoch": 0.6428318847592638,
|
|
"grad_norm": 6.493889097907155,
|
|
"learning_rate": 1.703954445736036e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.383,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17270
|
|
},
|
|
{
|
|
"epoch": 0.6432041093595876,
|
|
"grad_norm": 8.135436794172705,
|
|
"learning_rate": 1.7008757966885368e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3847,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17280
|
|
},
|
|
{
|
|
"epoch": 0.6435763339599114,
|
|
"grad_norm": 7.72947463548845,
|
|
"learning_rate": 1.6977984968313735e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3624,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17290
|
|
},
|
|
{
|
|
"epoch": 0.6439485585602353,
|
|
"grad_norm": 7.496020440463669,
|
|
"learning_rate": 1.6947225513600646e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17300
|
|
},
|
|
{
|
|
"epoch": 0.6443207831605591,
|
|
"grad_norm": 17.106954113085084,
|
|
"learning_rate": 1.6916479654678417e-07,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3785,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17310
|
|
},
|
|
{
|
|
"epoch": 0.6446930077608829,
|
|
"grad_norm": 5.740291744055549,
|
|
"learning_rate": 1.6885747443456362e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3697,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17320
|
|
},
|
|
{
|
|
"epoch": 0.6450652323612067,
|
|
"grad_norm": 7.245527066816409,
|
|
"learning_rate": 1.6855028931820826e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.351,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17330
|
|
},
|
|
{
|
|
"epoch": 0.6454374569615305,
|
|
"grad_norm": 7.120391586541118,
|
|
"learning_rate": 1.6824324171634946e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.38,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 17340
|
|
},
|
|
{
|
|
"epoch": 0.6458096815618545,
|
|
"grad_norm": 7.062738024833141,
|
|
"learning_rate": 1.679363321473871e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17350
|
|
},
|
|
{
|
|
"epoch": 0.6461819061621783,
|
|
"grad_norm": 9.596910794853457,
|
|
"learning_rate": 1.6762956112948756e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4082,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17360
|
|
},
|
|
{
|
|
"epoch": 0.6465541307625021,
|
|
"grad_norm": 6.954094964576264,
|
|
"learning_rate": 1.6732292918058356e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3867,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17370
|
|
},
|
|
{
|
|
"epoch": 0.6469263553628259,
|
|
"grad_norm": 7.263571871393878,
|
|
"learning_rate": 1.670164368183728e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3655,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17380
|
|
},
|
|
{
|
|
"epoch": 0.6472985799631498,
|
|
"grad_norm": 5.864324504141638,
|
|
"learning_rate": 1.6671008456031746e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3797,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17390
|
|
},
|
|
{
|
|
"epoch": 0.6476708045634736,
|
|
"grad_norm": 6.370103811304041,
|
|
"learning_rate": 1.6640387292364337e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17400
|
|
},
|
|
{
|
|
"epoch": 0.6480430291637974,
|
|
"grad_norm": 7.136151085394891,
|
|
"learning_rate": 1.6609780242533845e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3599,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17410
|
|
},
|
|
{
|
|
"epoch": 0.6484152537641212,
|
|
"grad_norm": 7.152215958755084,
|
|
"learning_rate": 1.657918735821528e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17420
|
|
},
|
|
{
|
|
"epoch": 0.6487874783644451,
|
|
"grad_norm": 6.608479050595447,
|
|
"learning_rate": 1.6548608691059702e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.4128,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 17430
|
|
},
|
|
{
|
|
"epoch": 0.649159702964769,
|
|
"grad_norm": 7.184198356054845,
|
|
"learning_rate": 1.6518044292694192e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3814,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17440
|
|
},
|
|
{
|
|
"epoch": 0.6495319275650928,
|
|
"grad_norm": 7.3849219166265785,
|
|
"learning_rate": 1.6487494214721726e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.4003,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 17450
|
|
},
|
|
{
|
|
"epoch": 0.6499041521654166,
|
|
"grad_norm": 7.350363988640259,
|
|
"learning_rate": 1.6456958508721102e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3727,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 17460
|
|
},
|
|
{
|
|
"epoch": 0.6502763767657405,
|
|
"grad_norm": 8.26428645466413,
|
|
"learning_rate": 1.6426437226246885e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3942,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 17470
|
|
},
|
|
{
|
|
"epoch": 0.6506486013660643,
|
|
"grad_norm": 9.309710688493814,
|
|
"learning_rate": 1.6395930418829228e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3891,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17480
|
|
},
|
|
{
|
|
"epoch": 0.6510208259663881,
|
|
"grad_norm": 7.8498678445650905,
|
|
"learning_rate": 1.63654381379739e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17490
|
|
},
|
|
{
|
|
"epoch": 0.6513930505667119,
|
|
"grad_norm": 7.535152618932254,
|
|
"learning_rate": 1.6334960435162112e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17500
|
|
},
|
|
{
|
|
"epoch": 0.6517652751670358,
|
|
"grad_norm": 7.358269895979216,
|
|
"learning_rate": 1.6304497361850483e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3676,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.0625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17510
|
|
},
|
|
{
|
|
"epoch": 0.6521374997673596,
|
|
"grad_norm": 8.561521762667526,
|
|
"learning_rate": 1.6274048969470912e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3724,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17520
|
|
},
|
|
{
|
|
"epoch": 0.6525097243676835,
|
|
"grad_norm": 7.665375864409572,
|
|
"learning_rate": 1.6243615309430535e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17530
|
|
},
|
|
{
|
|
"epoch": 0.6528819489680073,
|
|
"grad_norm": 7.74691659036126,
|
|
"learning_rate": 1.621319643311159e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.96875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17540
|
|
},
|
|
{
|
|
"epoch": 0.6532541735683312,
|
|
"grad_norm": 7.236509837489699,
|
|
"learning_rate": 1.618279239187138e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17550
|
|
},
|
|
{
|
|
"epoch": 0.653626398168655,
|
|
"grad_norm": 6.7209346925812525,
|
|
"learning_rate": 1.6152403237042145e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 17560
|
|
},
|
|
{
|
|
"epoch": 0.6539986227689788,
|
|
"grad_norm": 7.67617450805799,
|
|
"learning_rate": 1.6122029019930987e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3742,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17570
|
|
},
|
|
{
|
|
"epoch": 0.6543708473693026,
|
|
"grad_norm": 7.256380265874201,
|
|
"learning_rate": 1.609166979181981e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17580
|
|
},
|
|
{
|
|
"epoch": 0.6547430719696264,
|
|
"grad_norm": 6.721373666702093,
|
|
"learning_rate": 1.6061325603965193e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.376,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17590
|
|
},
|
|
{
|
|
"epoch": 0.6551152965699503,
|
|
"grad_norm": 6.919132453981522,
|
|
"learning_rate": 1.6030996507598327e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17600
|
|
},
|
|
{
|
|
"epoch": 0.6554875211702741,
|
|
"grad_norm": 8.476190707216295,
|
|
"learning_rate": 1.6000682553924922e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17610
|
|
},
|
|
{
|
|
"epoch": 0.655859745770598,
|
|
"grad_norm": 8.35658974751357,
|
|
"learning_rate": 1.5970383794125128e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17620
|
|
},
|
|
{
|
|
"epoch": 0.6562319703709218,
|
|
"grad_norm": 6.34120967758989,
|
|
"learning_rate": 1.5940100279353434e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17630
|
|
},
|
|
{
|
|
"epoch": 0.6566041949712457,
|
|
"grad_norm": 7.822479277407586,
|
|
"learning_rate": 1.5909832060738592e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17640
|
|
},
|
|
{
|
|
"epoch": 0.6569764195715695,
|
|
"grad_norm": 7.6245119236483205,
|
|
"learning_rate": 1.5879579189383549e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3589,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17650
|
|
},
|
|
{
|
|
"epoch": 0.6573486441718933,
|
|
"grad_norm": 6.703844555255469,
|
|
"learning_rate": 1.5849341716365306e-07,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17660
|
|
},
|
|
{
|
|
"epoch": 0.6577208687722171,
|
|
"grad_norm": 8.325137916698806,
|
|
"learning_rate": 1.5819119692734892e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17670
|
|
},
|
|
{
|
|
"epoch": 0.658093093372541,
|
|
"grad_norm": 8.483554166766936,
|
|
"learning_rate": 1.5788913169517237e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3757,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17680
|
|
},
|
|
{
|
|
"epoch": 0.6584653179728648,
|
|
"grad_norm": 7.895649569567779,
|
|
"learning_rate": 1.5758722197711122e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17690
|
|
},
|
|
{
|
|
"epoch": 0.6588375425731887,
|
|
"grad_norm": 6.671398947312763,
|
|
"learning_rate": 1.5728546828289042e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 17700
|
|
},
|
|
{
|
|
"epoch": 0.6592097671735125,
|
|
"grad_norm": 6.420889749284831,
|
|
"learning_rate": 1.5698387112197177e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 17710
|
|
},
|
|
{
|
|
"epoch": 0.6595819917738364,
|
|
"grad_norm": 7.614492339724009,
|
|
"learning_rate": 1.5668243100355262e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3803,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17720
|
|
},
|
|
{
|
|
"epoch": 0.6599542163741602,
|
|
"grad_norm": 6.8731322586511245,
|
|
"learning_rate": 1.5638114843656524e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3612,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17730
|
|
},
|
|
{
|
|
"epoch": 0.660326440974484,
|
|
"grad_norm": 6.9659614869355,
|
|
"learning_rate": 1.5608002392967603e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3698,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 17740
|
|
},
|
|
{
|
|
"epoch": 0.6606986655748078,
|
|
"grad_norm": 8.020420006283318,
|
|
"learning_rate": 1.5577905799128422e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3831,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17750
|
|
},
|
|
{
|
|
"epoch": 0.6610708901751317,
|
|
"grad_norm": 8.159946141811274,
|
|
"learning_rate": 1.5547825112952166e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4059,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17760
|
|
},
|
|
{
|
|
"epoch": 0.6614431147754555,
|
|
"grad_norm": 7.902452110400844,
|
|
"learning_rate": 1.551776038522513e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3993,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 17770
|
|
},
|
|
{
|
|
"epoch": 0.6618153393757793,
|
|
"grad_norm": 6.820410450554174,
|
|
"learning_rate": 1.54877116667067e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17780
|
|
},
|
|
{
|
|
"epoch": 0.6621875639761032,
|
|
"grad_norm": 5.841998354615362,
|
|
"learning_rate": 1.5457679008129203e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17790
|
|
},
|
|
{
|
|
"epoch": 0.6625597885764271,
|
|
"grad_norm": 7.401758951195473,
|
|
"learning_rate": 1.542766246019787e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17800
|
|
},
|
|
{
|
|
"epoch": 0.6629320131767509,
|
|
"grad_norm": 7.468672040865245,
|
|
"learning_rate": 1.539766207359073e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3829,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 17810
|
|
},
|
|
{
|
|
"epoch": 0.6633042377770747,
|
|
"grad_norm": 6.295832706880409,
|
|
"learning_rate": 1.5367677898958515e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17820
|
|
},
|
|
{
|
|
"epoch": 0.6636764623773985,
|
|
"grad_norm": 7.724661356118005,
|
|
"learning_rate": 1.53377099869246e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17830
|
|
},
|
|
{
|
|
"epoch": 0.6640486869777223,
|
|
"grad_norm": 6.697785630645924,
|
|
"learning_rate": 1.5307758388084884e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3857,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17840
|
|
},
|
|
{
|
|
"epoch": 0.6644209115780462,
|
|
"grad_norm": 8.052704348726998,
|
|
"learning_rate": 1.5277823153007754e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4105,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17850
|
|
},
|
|
{
|
|
"epoch": 0.66479313617837,
|
|
"grad_norm": 7.208057597359437,
|
|
"learning_rate": 1.5247904332233936e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17860
|
|
},
|
|
{
|
|
"epoch": 0.6651653607786938,
|
|
"grad_norm": 6.694927472297861,
|
|
"learning_rate": 1.521800197627647e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 17870
|
|
},
|
|
{
|
|
"epoch": 0.6655375853790177,
|
|
"grad_norm": 8.013538758189503,
|
|
"learning_rate": 1.5188116135620577e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4219,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 17880
|
|
},
|
|
{
|
|
"epoch": 0.6659098099793416,
|
|
"grad_norm": 7.339736998915164,
|
|
"learning_rate": 1.515824686072361e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17890
|
|
},
|
|
{
|
|
"epoch": 0.6662820345796654,
|
|
"grad_norm": 6.6280379890612195,
|
|
"learning_rate": 1.5128394202014948e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3762,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 17900
|
|
},
|
|
{
|
|
"epoch": 0.6666542591799892,
|
|
"grad_norm": 7.786534909266941,
|
|
"learning_rate": 1.5098558209895914e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3743,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 17910
|
|
},
|
|
{
|
|
"epoch": 0.667026483780313,
|
|
"grad_norm": 6.293455990372887,
|
|
"learning_rate": 1.50687389347397e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3695,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17920
|
|
},
|
|
{
|
|
"epoch": 0.6673987083806369,
|
|
"grad_norm": 7.8658922771663855,
|
|
"learning_rate": 1.5038936426891259e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 17930
|
|
},
|
|
{
|
|
"epoch": 0.6677709329809607,
|
|
"grad_norm": 7.742020479574208,
|
|
"learning_rate": 1.5009150736667247e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3741,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17940
|
|
},
|
|
{
|
|
"epoch": 0.6681431575812845,
|
|
"grad_norm": 6.924379641898376,
|
|
"learning_rate": 1.4979381914355928e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3823,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 17950
|
|
},
|
|
{
|
|
"epoch": 0.6685153821816083,
|
|
"grad_norm": 8.630914435030103,
|
|
"learning_rate": 1.4949630010217087e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17960
|
|
},
|
|
{
|
|
"epoch": 0.6688876067819323,
|
|
"grad_norm": 8.434126803700602,
|
|
"learning_rate": 1.491989507448192e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17970
|
|
},
|
|
{
|
|
"epoch": 0.6692598313822561,
|
|
"grad_norm": 8.195438937311083,
|
|
"learning_rate": 1.4890177157353017e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3713,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 17980
|
|
},
|
|
{
|
|
"epoch": 0.6696320559825799,
|
|
"grad_norm": 7.776089402102017,
|
|
"learning_rate": 1.486047630900421e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 17990
|
|
},
|
|
{
|
|
"epoch": 0.6700042805829037,
|
|
"grad_norm": 6.08508424055095,
|
|
"learning_rate": 1.4830792579580514e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3998,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18000
|
|
},
|
|
{
|
|
"epoch": 0.6703765051832276,
|
|
"grad_norm": 8.539860345806389,
|
|
"learning_rate": 1.4801126019198045e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3839,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18010
|
|
},
|
|
{
|
|
"epoch": 0.6707487297835514,
|
|
"grad_norm": 6.672736634900353,
|
|
"learning_rate": 1.4771476677943934e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3866,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 18020
|
|
},
|
|
{
|
|
"epoch": 0.6711209543838752,
|
|
"grad_norm": 8.128001850818436,
|
|
"learning_rate": 1.4741844605876241e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3882,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18030
|
|
},
|
|
{
|
|
"epoch": 0.671493178984199,
|
|
"grad_norm": 7.523150335782813,
|
|
"learning_rate": 1.4712229853023845e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18040
|
|
},
|
|
{
|
|
"epoch": 0.6718654035845228,
|
|
"grad_norm": 6.680477861129114,
|
|
"learning_rate": 1.468263246938643e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18050
|
|
},
|
|
{
|
|
"epoch": 0.6722376281848468,
|
|
"grad_norm": 7.320362721480975,
|
|
"learning_rate": 1.4653052504934327e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 18060
|
|
},
|
|
{
|
|
"epoch": 0.6726098527851706,
|
|
"grad_norm": 7.14912576203958,
|
|
"learning_rate": 1.4623490009608453e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3703,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18070
|
|
},
|
|
{
|
|
"epoch": 0.6729820773854944,
|
|
"grad_norm": 7.090310188632294,
|
|
"learning_rate": 1.4593945033320252e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4106,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18080
|
|
},
|
|
{
|
|
"epoch": 0.6733543019858182,
|
|
"grad_norm": 6.944741673860654,
|
|
"learning_rate": 1.4564417625951558e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18090
|
|
},
|
|
{
|
|
"epoch": 0.6737265265861421,
|
|
"grad_norm": 8.430773797553202,
|
|
"learning_rate": 1.4534907837354577e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3814,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18100
|
|
},
|
|
{
|
|
"epoch": 0.6740987511864659,
|
|
"grad_norm": 7.357586980892462,
|
|
"learning_rate": 1.4505415717351752e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.383,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 18110
|
|
},
|
|
{
|
|
"epoch": 0.6744709757867897,
|
|
"grad_norm": 7.3917228660501015,
|
|
"learning_rate": 1.4475941315735704e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3731,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18120
|
|
},
|
|
{
|
|
"epoch": 0.6748432003871135,
|
|
"grad_norm": 8.75397377362964,
|
|
"learning_rate": 1.4446484682269117e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3908,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18130
|
|
},
|
|
{
|
|
"epoch": 0.6752154249874375,
|
|
"grad_norm": 6.777056543312223,
|
|
"learning_rate": 1.4417045866684698e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4008,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18140
|
|
},
|
|
{
|
|
"epoch": 0.6755876495877613,
|
|
"grad_norm": 7.495251145728057,
|
|
"learning_rate": 1.4387624918685072e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3935,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18150
|
|
},
|
|
{
|
|
"epoch": 0.6759598741880851,
|
|
"grad_norm": 6.804400726816325,
|
|
"learning_rate": 1.4358221887942684e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.371,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18160
|
|
},
|
|
{
|
|
"epoch": 0.6763320987884089,
|
|
"grad_norm": 6.848219741944667,
|
|
"learning_rate": 1.432883682409975e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18170
|
|
},
|
|
{
|
|
"epoch": 0.6767043233887328,
|
|
"grad_norm": 7.455849447925066,
|
|
"learning_rate": 1.4299469776768116e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.405,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18180
|
|
},
|
|
{
|
|
"epoch": 0.6770765479890566,
|
|
"grad_norm": 7.014838630952778,
|
|
"learning_rate": 1.4270120795529245e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18190
|
|
},
|
|
{
|
|
"epoch": 0.6774487725893804,
|
|
"grad_norm": 7.5398969910541,
|
|
"learning_rate": 1.424078992993408e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4015,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18200
|
|
},
|
|
{
|
|
"epoch": 0.6778209971897042,
|
|
"grad_norm": 8.495485978748052,
|
|
"learning_rate": 1.4211477229502996e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.3856,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.15625,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 18210
|
|
},
|
|
{
|
|
"epoch": 0.6781932217900281,
|
|
"grad_norm": 7.85193228022331,
|
|
"learning_rate": 1.418218274372567e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 18220
|
|
},
|
|
{
|
|
"epoch": 0.678565446390352,
|
|
"grad_norm": 6.753732327381483,
|
|
"learning_rate": 1.4152906522061047e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18230
|
|
},
|
|
{
|
|
"epoch": 0.6789376709906758,
|
|
"grad_norm": 7.392421065059972,
|
|
"learning_rate": 1.412364861393724e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3862,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18240
|
|
},
|
|
{
|
|
"epoch": 0.6793098955909996,
|
|
"grad_norm": 8.452852656709055,
|
|
"learning_rate": 1.4094409068751428e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18250
|
|
},
|
|
{
|
|
"epoch": 0.6796821201913235,
|
|
"grad_norm": 8.630750766088273,
|
|
"learning_rate": 1.406518793586981e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18260
|
|
},
|
|
{
|
|
"epoch": 0.6800543447916473,
|
|
"grad_norm": 8.477951416403787,
|
|
"learning_rate": 1.4035985264627456e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18270
|
|
},
|
|
{
|
|
"epoch": 0.6804265693919711,
|
|
"grad_norm": 7.168790130537178,
|
|
"learning_rate": 1.400680110432831e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3845,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18280
|
|
},
|
|
{
|
|
"epoch": 0.6807987939922949,
|
|
"grad_norm": 7.2309897357925905,
|
|
"learning_rate": 1.3977635504245047e-07,
|
|
"logits/chosen": -3.5625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3605,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18290
|
|
},
|
|
{
|
|
"epoch": 0.6811710185926187,
|
|
"grad_norm": 8.687017489276187,
|
|
"learning_rate": 1.394848851361901e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18300
|
|
},
|
|
{
|
|
"epoch": 0.6815432431929426,
|
|
"grad_norm": 8.218406367305924,
|
|
"learning_rate": 1.3919360181660107e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3988,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18310
|
|
},
|
|
{
|
|
"epoch": 0.6819154677932665,
|
|
"grad_norm": 9.441751194250308,
|
|
"learning_rate": 1.3890250557546763e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3854,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18320
|
|
},
|
|
{
|
|
"epoch": 0.6822876923935903,
|
|
"grad_norm": 8.62585075037886,
|
|
"learning_rate": 1.3861159690425806e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.4001,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 18330
|
|
},
|
|
{
|
|
"epoch": 0.6826599169939142,
|
|
"grad_norm": 8.292908793977418,
|
|
"learning_rate": 1.3832087629412402e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3849,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18340
|
|
},
|
|
{
|
|
"epoch": 0.683032141594238,
|
|
"grad_norm": 7.483825971602488,
|
|
"learning_rate": 1.3803034423589982e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18350
|
|
},
|
|
{
|
|
"epoch": 0.6834043661945618,
|
|
"grad_norm": 6.5419504223210945,
|
|
"learning_rate": 1.37740001220101e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18360
|
|
},
|
|
{
|
|
"epoch": 0.6837765907948856,
|
|
"grad_norm": 7.399141591643264,
|
|
"learning_rate": 1.3744984773692425e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 18370
|
|
},
|
|
{
|
|
"epoch": 0.6841488153952094,
|
|
"grad_norm": 7.574729790367357,
|
|
"learning_rate": 1.3715988427624632e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18380
|
|
},
|
|
{
|
|
"epoch": 0.6845210399955333,
|
|
"grad_norm": 8.336569369049675,
|
|
"learning_rate": 1.3687011132762288e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3758,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18390
|
|
},
|
|
{
|
|
"epoch": 0.6848932645958571,
|
|
"grad_norm": 9.116011563173396,
|
|
"learning_rate": 1.3658052938028834e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18400
|
|
},
|
|
{
|
|
"epoch": 0.685265489196181,
|
|
"grad_norm": 7.361084218418948,
|
|
"learning_rate": 1.362911389231541e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18410
|
|
},
|
|
{
|
|
"epoch": 0.6856377137965048,
|
|
"grad_norm": 7.1088981538894735,
|
|
"learning_rate": 1.3600194044480866e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4102,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18420
|
|
},
|
|
{
|
|
"epoch": 0.6860099383968287,
|
|
"grad_norm": 8.453465401972265,
|
|
"learning_rate": 1.3571293443351627e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4015,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 18430
|
|
},
|
|
{
|
|
"epoch": 0.6863821629971525,
|
|
"grad_norm": 6.9623456220842055,
|
|
"learning_rate": 1.3542412137721643e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18440
|
|
},
|
|
{
|
|
"epoch": 0.6867543875974763,
|
|
"grad_norm": 7.373753015699395,
|
|
"learning_rate": 1.351355017635224e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4051,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 18450
|
|
},
|
|
{
|
|
"epoch": 0.6871266121978001,
|
|
"grad_norm": 6.110682753310095,
|
|
"learning_rate": 1.3484707607972134e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3683,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18460
|
|
},
|
|
{
|
|
"epoch": 0.687498836798124,
|
|
"grad_norm": 6.216862887844371,
|
|
"learning_rate": 1.3455884481277253e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3658,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 18470
|
|
},
|
|
{
|
|
"epoch": 0.6878710613984478,
|
|
"grad_norm": 7.843228215837358,
|
|
"learning_rate": 1.342708084493075e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3536,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 18480
|
|
},
|
|
{
|
|
"epoch": 0.6882432859987716,
|
|
"grad_norm": 8.177147552946252,
|
|
"learning_rate": 1.339829674756285e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3881,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18490
|
|
},
|
|
{
|
|
"epoch": 0.6886155105990955,
|
|
"grad_norm": 9.091396570676634,
|
|
"learning_rate": 1.336953223777077e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.368,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18500
|
|
},
|
|
{
|
|
"epoch": 0.6889877351994194,
|
|
"grad_norm": 8.41485134276105,
|
|
"learning_rate": 1.3340787364118688e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.4121,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 18510
|
|
},
|
|
{
|
|
"epoch": 0.6893599597997432,
|
|
"grad_norm": 8.133577540422605,
|
|
"learning_rate": 1.331206217513759e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18520
|
|
},
|
|
{
|
|
"epoch": 0.689732184400067,
|
|
"grad_norm": 8.773817689038646,
|
|
"learning_rate": 1.328335671932529e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.4069,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18530
|
|
},
|
|
{
|
|
"epoch": 0.6901044090003908,
|
|
"grad_norm": 8.08254537916329,
|
|
"learning_rate": 1.3254671045146222e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 18540
|
|
},
|
|
{
|
|
"epoch": 0.6904766336007147,
|
|
"grad_norm": 7.365006900092196,
|
|
"learning_rate": 1.322600520103146e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -896.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4078,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18550
|
|
},
|
|
{
|
|
"epoch": 0.6908488582010385,
|
|
"grad_norm": 9.325093303002477,
|
|
"learning_rate": 1.319735923537857e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18560
|
|
},
|
|
{
|
|
"epoch": 0.6912210828013623,
|
|
"grad_norm": 7.143969172856071,
|
|
"learning_rate": 1.3168733196551596e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3908,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18570
|
|
},
|
|
{
|
|
"epoch": 0.6915933074016862,
|
|
"grad_norm": 6.913197392991485,
|
|
"learning_rate": 1.314012713288092e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3751,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18580
|
|
},
|
|
{
|
|
"epoch": 0.69196553200201,
|
|
"grad_norm": 7.300134507003808,
|
|
"learning_rate": 1.3111541092663168e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3864,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18590
|
|
},
|
|
{
|
|
"epoch": 0.6923377566023339,
|
|
"grad_norm": 5.865880795509503,
|
|
"learning_rate": 1.308297512416121e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 18600
|
|
},
|
|
{
|
|
"epoch": 0.6927099812026577,
|
|
"grad_norm": 6.5150462879030755,
|
|
"learning_rate": 1.3054429275603983e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18610
|
|
},
|
|
{
|
|
"epoch": 0.6930822058029815,
|
|
"grad_norm": 6.162229749248615,
|
|
"learning_rate": 1.302590359518651e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18620
|
|
},
|
|
{
|
|
"epoch": 0.6934544304033053,
|
|
"grad_norm": 7.112979917809726,
|
|
"learning_rate": 1.29973981310697e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18630
|
|
},
|
|
{
|
|
"epoch": 0.6938266550036292,
|
|
"grad_norm": 7.930354256054114,
|
|
"learning_rate": 1.2968912931380375e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3881,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 18640
|
|
},
|
|
{
|
|
"epoch": 0.694198879603953,
|
|
"grad_norm": 8.283602459763573,
|
|
"learning_rate": 1.2940448044211134e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18650
|
|
},
|
|
{
|
|
"epoch": 0.6945711042042768,
|
|
"grad_norm": 9.925017251038437,
|
|
"learning_rate": 1.2912003517620252e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18660
|
|
},
|
|
{
|
|
"epoch": 0.6949433288046007,
|
|
"grad_norm": 7.7395037683956645,
|
|
"learning_rate": 1.2883579399631689e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 18670
|
|
},
|
|
{
|
|
"epoch": 0.6953155534049246,
|
|
"grad_norm": 7.5312220260699965,
|
|
"learning_rate": 1.2855175738234886e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3714,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18680
|
|
},
|
|
{
|
|
"epoch": 0.6956877780052484,
|
|
"grad_norm": 8.143291469487965,
|
|
"learning_rate": 1.282679258138479e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3694,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18690
|
|
},
|
|
{
|
|
"epoch": 0.6960600026055722,
|
|
"grad_norm": 10.100651517895768,
|
|
"learning_rate": 1.27984299770017e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3782,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18700
|
|
},
|
|
{
|
|
"epoch": 0.696432227205896,
|
|
"grad_norm": 8.116546936064687,
|
|
"learning_rate": 1.277008797297123e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18710
|
|
},
|
|
{
|
|
"epoch": 0.6968044518062199,
|
|
"grad_norm": 6.923805127350405,
|
|
"learning_rate": 1.2741766617144218e-07,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 18720
|
|
},
|
|
{
|
|
"epoch": 0.6971766764065437,
|
|
"grad_norm": 8.267439727499852,
|
|
"learning_rate": 1.2713465957336633e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 18730
|
|
},
|
|
{
|
|
"epoch": 0.6975489010068675,
|
|
"grad_norm": 8.29436107175343,
|
|
"learning_rate": 1.268518604132951e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.396,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18740
|
|
},
|
|
{
|
|
"epoch": 0.6979211256071913,
|
|
"grad_norm": 8.416412598110844,
|
|
"learning_rate": 1.2656926916868833e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3857,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18750
|
|
},
|
|
{
|
|
"epoch": 0.6982933502075153,
|
|
"grad_norm": 7.46882316518551,
|
|
"learning_rate": 1.262868863166554e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.381,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18760
|
|
},
|
|
{
|
|
"epoch": 0.6986655748078391,
|
|
"grad_norm": 7.4992162791207795,
|
|
"learning_rate": 1.260047123339532e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4114,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18770
|
|
},
|
|
{
|
|
"epoch": 0.6990377994081629,
|
|
"grad_norm": 7.459310900290135,
|
|
"learning_rate": 1.2572274769698654e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4136,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18780
|
|
},
|
|
{
|
|
"epoch": 0.6994100240084867,
|
|
"grad_norm": 8.210534639475684,
|
|
"learning_rate": 1.2544099288180626e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18790
|
|
},
|
|
{
|
|
"epoch": 0.6997822486088106,
|
|
"grad_norm": 8.96233676110681,
|
|
"learning_rate": 1.2515944836410942e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4082,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 18800
|
|
},
|
|
{
|
|
"epoch": 0.7001544732091344,
|
|
"grad_norm": 7.421784461654145,
|
|
"learning_rate": 1.248781146192377e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -884.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3838,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 18810
|
|
},
|
|
{
|
|
"epoch": 0.7005266978094582,
|
|
"grad_norm": 9.497109180788014,
|
|
"learning_rate": 1.2459699212217713e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.25,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18820
|
|
},
|
|
{
|
|
"epoch": 0.700898922409782,
|
|
"grad_norm": 7.651581202495108,
|
|
"learning_rate": 1.2431608134755712e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3804,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18830
|
|
},
|
|
{
|
|
"epoch": 0.7012711470101058,
|
|
"grad_norm": 8.231624309618654,
|
|
"learning_rate": 1.2403538276964926e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3859,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18840
|
|
},
|
|
{
|
|
"epoch": 0.7016433716104298,
|
|
"grad_norm": 7.5632523843286,
|
|
"learning_rate": 1.2375489686236724e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3908,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18850
|
|
},
|
|
{
|
|
"epoch": 0.7020155962107536,
|
|
"grad_norm": 7.664421022077882,
|
|
"learning_rate": 1.2347462409926556e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3929,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18860
|
|
},
|
|
{
|
|
"epoch": 0.7023878208110774,
|
|
"grad_norm": 7.1432555849468065,
|
|
"learning_rate": 1.23194564953539e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3959,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18870
|
|
},
|
|
{
|
|
"epoch": 0.7027600454114012,
|
|
"grad_norm": 6.829591861062921,
|
|
"learning_rate": 1.229147198980214e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3525,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18880
|
|
},
|
|
{
|
|
"epoch": 0.7031322700117251,
|
|
"grad_norm": 7.9527755493703065,
|
|
"learning_rate": 1.2263508940518532e-07,
|
|
"logits/chosen": -3.59375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18890
|
|
},
|
|
{
|
|
"epoch": 0.7035044946120489,
|
|
"grad_norm": 9.706660173223497,
|
|
"learning_rate": 1.2235567394714112e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18900
|
|
},
|
|
{
|
|
"epoch": 0.7038767192123727,
|
|
"grad_norm": 7.504170788777044,
|
|
"learning_rate": 1.2207647399563597e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3685,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18910
|
|
},
|
|
{
|
|
"epoch": 0.7042489438126965,
|
|
"grad_norm": 7.806128548464252,
|
|
"learning_rate": 1.2179749002205342e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18920
|
|
},
|
|
{
|
|
"epoch": 0.7046211684130205,
|
|
"grad_norm": 8.414685566486376,
|
|
"learning_rate": 1.21518722497412e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3602,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 18930
|
|
},
|
|
{
|
|
"epoch": 0.7049933930133443,
|
|
"grad_norm": 8.206639837018972,
|
|
"learning_rate": 1.212401718923651e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3712,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 18940
|
|
},
|
|
{
|
|
"epoch": 0.7053656176136681,
|
|
"grad_norm": 7.464994509703324,
|
|
"learning_rate": 1.2096183867719981e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.390625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3931,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 18950
|
|
},
|
|
{
|
|
"epoch": 0.7057378422139919,
|
|
"grad_norm": 7.073297798398018,
|
|
"learning_rate": 1.206837233218363e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.366,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18960
|
|
},
|
|
{
|
|
"epoch": 0.7061100668143158,
|
|
"grad_norm": 7.312628600203564,
|
|
"learning_rate": 1.2040582629582658e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3783,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 18970
|
|
},
|
|
{
|
|
"epoch": 0.7064822914146396,
|
|
"grad_norm": 7.827428793173597,
|
|
"learning_rate": 1.2012814806835432e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4065,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 18980
|
|
},
|
|
{
|
|
"epoch": 0.7068545160149634,
|
|
"grad_norm": 7.848155482470442,
|
|
"learning_rate": 1.198506891082338e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3954,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 18990
|
|
},
|
|
{
|
|
"epoch": 0.7072267406152872,
|
|
"grad_norm": 7.129579090884928,
|
|
"learning_rate": 1.1957344988390903e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3906,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19000
|
|
},
|
|
{
|
|
"epoch": 0.707598965215611,
|
|
"grad_norm": 8.852683902988014,
|
|
"learning_rate": 1.192964308634531e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3827,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19010
|
|
},
|
|
{
|
|
"epoch": 0.707971189815935,
|
|
"grad_norm": 7.599536487634325,
|
|
"learning_rate": 1.1901963251456704e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3909,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19020
|
|
},
|
|
{
|
|
"epoch": 0.7083434144162588,
|
|
"grad_norm": 9.840114067389997,
|
|
"learning_rate": 1.1874305530457967e-07,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19030
|
|
},
|
|
{
|
|
"epoch": 0.7087156390165826,
|
|
"grad_norm": 6.858273063559076,
|
|
"learning_rate": 1.1846669970044629e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3636,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19040
|
|
},
|
|
{
|
|
"epoch": 0.7090878636169065,
|
|
"grad_norm": 8.205029910576494,
|
|
"learning_rate": 1.181905661687482e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19050
|
|
},
|
|
{
|
|
"epoch": 0.7094600882172303,
|
|
"grad_norm": 6.7870638339259814,
|
|
"learning_rate": 1.179146551756914e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 19060
|
|
},
|
|
{
|
|
"epoch": 0.7098323128175541,
|
|
"grad_norm": 8.961093141651975,
|
|
"learning_rate": 1.1763896718710656e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3675,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19070
|
|
},
|
|
{
|
|
"epoch": 0.7102045374178779,
|
|
"grad_norm": 7.4041464457521915,
|
|
"learning_rate": 1.1736350266844766e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3951,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19080
|
|
},
|
|
{
|
|
"epoch": 0.7105767620182017,
|
|
"grad_norm": 8.607797854949952,
|
|
"learning_rate": 1.1708826208479145e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4072,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19090
|
|
},
|
|
{
|
|
"epoch": 0.7109489866185256,
|
|
"grad_norm": 6.993832561903155,
|
|
"learning_rate": 1.1681324590083663e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3706,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19100
|
|
},
|
|
{
|
|
"epoch": 0.7113212112188495,
|
|
"grad_norm": 7.718634524647451,
|
|
"learning_rate": 1.1653845458090286e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3732,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19110
|
|
},
|
|
{
|
|
"epoch": 0.7116934358191733,
|
|
"grad_norm": 7.5425470055642,
|
|
"learning_rate": 1.1626388858893033e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3718,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19120
|
|
},
|
|
{
|
|
"epoch": 0.7120656604194971,
|
|
"grad_norm": 8.790345727244683,
|
|
"learning_rate": 1.1598954838847877e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3987,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19130
|
|
},
|
|
{
|
|
"epoch": 0.712437885019821,
|
|
"grad_norm": 6.671980153724739,
|
|
"learning_rate": 1.1571543444272674e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19140
|
|
},
|
|
{
|
|
"epoch": 0.7128101096201448,
|
|
"grad_norm": 7.51968124366577,
|
|
"learning_rate": 1.1544154721447058e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3823,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19150
|
|
},
|
|
{
|
|
"epoch": 0.7131823342204686,
|
|
"grad_norm": 6.868266559921231,
|
|
"learning_rate": 1.1516788716612413e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3709,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19160
|
|
},
|
|
{
|
|
"epoch": 0.7135545588207924,
|
|
"grad_norm": 6.868201983880112,
|
|
"learning_rate": 1.1489445475971751e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3977,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 19170
|
|
},
|
|
{
|
|
"epoch": 0.7139267834211163,
|
|
"grad_norm": 6.670194131632748,
|
|
"learning_rate": 1.1462125045689661e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3678,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.1875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 19180
|
|
},
|
|
{
|
|
"epoch": 0.7142990080214401,
|
|
"grad_norm": 8.427040296471885,
|
|
"learning_rate": 1.1434827471892222e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.377,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19190
|
|
},
|
|
{
|
|
"epoch": 0.714671232621764,
|
|
"grad_norm": 18.245487151102406,
|
|
"learning_rate": 1.1407552800666895e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19200
|
|
},
|
|
{
|
|
"epoch": 0.7150434572220878,
|
|
"grad_norm": 7.09631582677767,
|
|
"learning_rate": 1.138030107806251e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.3965,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 19210
|
|
},
|
|
{
|
|
"epoch": 0.7154156818224117,
|
|
"grad_norm": 7.8701890292837176,
|
|
"learning_rate": 1.1353072350089135e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4156,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19220
|
|
},
|
|
{
|
|
"epoch": 0.7157879064227355,
|
|
"grad_norm": 7.649791312300932,
|
|
"learning_rate": 1.1325866662718025e-07,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3855,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19230
|
|
},
|
|
{
|
|
"epoch": 0.7161601310230593,
|
|
"grad_norm": 7.297548504293173,
|
|
"learning_rate": 1.129868406188151e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3771,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19240
|
|
},
|
|
{
|
|
"epoch": 0.7165323556233831,
|
|
"grad_norm": 7.626565928290019,
|
|
"learning_rate": 1.1271524593472973e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -1000.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19250
|
|
},
|
|
{
|
|
"epoch": 0.716904580223707,
|
|
"grad_norm": 8.363082445240961,
|
|
"learning_rate": 1.1244388303346722e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19260
|
|
},
|
|
{
|
|
"epoch": 0.7172768048240308,
|
|
"grad_norm": 6.936392824292014,
|
|
"learning_rate": 1.1217275237317942e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19270
|
|
},
|
|
{
|
|
"epoch": 0.7176490294243546,
|
|
"grad_norm": 8.338909404728245,
|
|
"learning_rate": 1.1190185441162613e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19280
|
|
},
|
|
{
|
|
"epoch": 0.7180212540246785,
|
|
"grad_norm": 7.620053477052517,
|
|
"learning_rate": 1.1163118960617402e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4044,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19290
|
|
},
|
|
{
|
|
"epoch": 0.7183934786250024,
|
|
"grad_norm": 8.613803768768447,
|
|
"learning_rate": 1.1136075841379636e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19300
|
|
},
|
|
{
|
|
"epoch": 0.7187657032253262,
|
|
"grad_norm": 7.97601546847158,
|
|
"learning_rate": 1.1109056129107192e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19310
|
|
},
|
|
{
|
|
"epoch": 0.71913792782565,
|
|
"grad_norm": 8.165563836344388,
|
|
"learning_rate": 1.1082059869418428e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19320
|
|
},
|
|
{
|
|
"epoch": 0.7195101524259738,
|
|
"grad_norm": 8.413152276271282,
|
|
"learning_rate": 1.1055087107892123e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3819,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19330
|
|
},
|
|
{
|
|
"epoch": 0.7198823770262976,
|
|
"grad_norm": 7.261766176925426,
|
|
"learning_rate": 1.102813789006734e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19340
|
|
},
|
|
{
|
|
"epoch": 0.7202546016266215,
|
|
"grad_norm": 8.272709301165099,
|
|
"learning_rate": 1.1001212261443435e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3597,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19350
|
|
},
|
|
{
|
|
"epoch": 0.7206268262269453,
|
|
"grad_norm": 8.001438607259361,
|
|
"learning_rate": 1.0974310267479919e-07,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4001,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19360
|
|
},
|
|
{
|
|
"epoch": 0.7209990508272692,
|
|
"grad_norm": 6.876370749996694,
|
|
"learning_rate": 1.0947431953596412e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19370
|
|
},
|
|
{
|
|
"epoch": 0.721371275427593,
|
|
"grad_norm": 8.588236052407888,
|
|
"learning_rate": 1.0920577365172528e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19380
|
|
},
|
|
{
|
|
"epoch": 0.7217435000279169,
|
|
"grad_norm": 9.550721696908235,
|
|
"learning_rate": 1.0893746547547861e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3851,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19390
|
|
},
|
|
{
|
|
"epoch": 0.7221157246282407,
|
|
"grad_norm": 8.50418662448344,
|
|
"learning_rate": 1.0866939546021822e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3966,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19400
|
|
},
|
|
{
|
|
"epoch": 0.7224879492285645,
|
|
"grad_norm": 6.8052460079379085,
|
|
"learning_rate": 1.084015640585367e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19410
|
|
},
|
|
{
|
|
"epoch": 0.7228601738288883,
|
|
"grad_norm": 7.98772118942524,
|
|
"learning_rate": 1.0813397172262354e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19420
|
|
},
|
|
{
|
|
"epoch": 0.7232323984292122,
|
|
"grad_norm": 7.680205100622341,
|
|
"learning_rate": 1.0786661890426436e-07,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19430
|
|
},
|
|
{
|
|
"epoch": 0.723604623029536,
|
|
"grad_norm": 8.739911337143017,
|
|
"learning_rate": 1.0759950605484078e-07,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19440
|
|
},
|
|
{
|
|
"epoch": 0.7239768476298598,
|
|
"grad_norm": 7.330881464875703,
|
|
"learning_rate": 1.0733263362532885e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4027,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19450
|
|
},
|
|
{
|
|
"epoch": 0.7243490722301837,
|
|
"grad_norm": 9.453283041399086,
|
|
"learning_rate": 1.0706600206629931e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3808,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19460
|
|
},
|
|
{
|
|
"epoch": 0.7247212968305076,
|
|
"grad_norm": 7.4133289804267255,
|
|
"learning_rate": 1.0679961182791561e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19470
|
|
},
|
|
{
|
|
"epoch": 0.7250935214308314,
|
|
"grad_norm": 8.054795760370327,
|
|
"learning_rate": 1.0653346335993424e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19480
|
|
},
|
|
{
|
|
"epoch": 0.7254657460311552,
|
|
"grad_norm": 7.556716188852257,
|
|
"learning_rate": 1.0626755711170302e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.386,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19490
|
|
},
|
|
{
|
|
"epoch": 0.725837970631479,
|
|
"grad_norm": 6.929064151601486,
|
|
"learning_rate": 1.0600189353216132e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3883,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19500
|
|
},
|
|
{
|
|
"epoch": 0.7262101952318029,
|
|
"grad_norm": 7.583257761154129,
|
|
"learning_rate": 1.0573647306983866e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19510
|
|
},
|
|
{
|
|
"epoch": 0.7265824198321267,
|
|
"grad_norm": 7.14997308400207,
|
|
"learning_rate": 1.0547129617285377e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3798,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19520
|
|
},
|
|
{
|
|
"epoch": 0.7269546444324505,
|
|
"grad_norm": 7.539902119396339,
|
|
"learning_rate": 1.0520636328891467e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3819,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19530
|
|
},
|
|
{
|
|
"epoch": 0.7273268690327743,
|
|
"grad_norm": 8.104055162326393,
|
|
"learning_rate": 1.0494167486531677e-07,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3546,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19540
|
|
},
|
|
{
|
|
"epoch": 0.7276990936330983,
|
|
"grad_norm": 8.274996611232103,
|
|
"learning_rate": 1.0467723134894357e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19550
|
|
},
|
|
{
|
|
"epoch": 0.7280713182334221,
|
|
"grad_norm": 7.6047273258660155,
|
|
"learning_rate": 1.0441303318626434e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19560
|
|
},
|
|
{
|
|
"epoch": 0.7284435428337459,
|
|
"grad_norm": 7.820628088755584,
|
|
"learning_rate": 1.0414908082333451e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 19570
|
|
},
|
|
{
|
|
"epoch": 0.7288157674340697,
|
|
"grad_norm": 8.668359508988418,
|
|
"learning_rate": 1.0388537470579448e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19580
|
|
},
|
|
{
|
|
"epoch": 0.7291879920343936,
|
|
"grad_norm": 7.92161587949906,
|
|
"learning_rate": 1.0362191527886857e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 19590
|
|
},
|
|
{
|
|
"epoch": 0.7295602166347174,
|
|
"grad_norm": 8.26708040539958,
|
|
"learning_rate": 1.0335870298736526e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 19600
|
|
},
|
|
{
|
|
"epoch": 0.7299324412350412,
|
|
"grad_norm": 8.466807479832728,
|
|
"learning_rate": 1.030957382756751e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19610
|
|
},
|
|
{
|
|
"epoch": 0.730304665835365,
|
|
"grad_norm": 6.604610647023422,
|
|
"learning_rate": 1.0283302158777113e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19620
|
|
},
|
|
{
|
|
"epoch": 0.7306768904356888,
|
|
"grad_norm": 6.578293423749665,
|
|
"learning_rate": 1.0257055336720726e-07,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3664,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19630
|
|
},
|
|
{
|
|
"epoch": 0.7310491150360128,
|
|
"grad_norm": 7.911281575597875,
|
|
"learning_rate": 1.0230833405711822e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3631,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19640
|
|
},
|
|
{
|
|
"epoch": 0.7314213396363366,
|
|
"grad_norm": 7.1298048042315,
|
|
"learning_rate": 1.020463641002183e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3764,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19650
|
|
},
|
|
{
|
|
"epoch": 0.7317935642366604,
|
|
"grad_norm": 7.9157416083729695,
|
|
"learning_rate": 1.0178464393880093e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19660
|
|
},
|
|
{
|
|
"epoch": 0.7321657888369842,
|
|
"grad_norm": 8.116269489251481,
|
|
"learning_rate": 1.0152317401473782e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19670
|
|
},
|
|
{
|
|
"epoch": 0.7325380134373081,
|
|
"grad_norm": 9.046356608643071,
|
|
"learning_rate": 1.0126195476947779e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19680
|
|
},
|
|
{
|
|
"epoch": 0.7329102380376319,
|
|
"grad_norm": 9.314054867577987,
|
|
"learning_rate": 1.0100098664404719e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3837,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19690
|
|
},
|
|
{
|
|
"epoch": 0.7332824626379557,
|
|
"grad_norm": 7.210939458509012,
|
|
"learning_rate": 1.0074027007904764e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19700
|
|
},
|
|
{
|
|
"epoch": 0.7336546872382795,
|
|
"grad_norm": 7.33360645090443,
|
|
"learning_rate": 1.0047980551465653e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19710
|
|
},
|
|
{
|
|
"epoch": 0.7340269118386035,
|
|
"grad_norm": 6.942503999823571,
|
|
"learning_rate": 1.002195933906255e-07,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3685,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19720
|
|
},
|
|
{
|
|
"epoch": 0.7343991364389273,
|
|
"grad_norm": 7.07266298619708,
|
|
"learning_rate": 9.995963414628011e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3803,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19730
|
|
},
|
|
{
|
|
"epoch": 0.7347713610392511,
|
|
"grad_norm": 7.709944510841025,
|
|
"learning_rate": 9.969992822051904e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3774,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19740
|
|
},
|
|
{
|
|
"epoch": 0.7351435856395749,
|
|
"grad_norm": 10.178383045776142,
|
|
"learning_rate": 9.944047605181319e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3794,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19750
|
|
},
|
|
{
|
|
"epoch": 0.7355158102398988,
|
|
"grad_norm": 7.861546356540952,
|
|
"learning_rate": 9.918127807820512e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3702,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 19760
|
|
},
|
|
{
|
|
"epoch": 0.7358880348402226,
|
|
"grad_norm": 7.974467515975566,
|
|
"learning_rate": 9.892233473730799e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3951,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19770
|
|
},
|
|
{
|
|
"epoch": 0.7362602594405464,
|
|
"grad_norm": 7.135480767410617,
|
|
"learning_rate": 9.866364646630531e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3858,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 19780
|
|
},
|
|
{
|
|
"epoch": 0.7366324840408702,
|
|
"grad_norm": 8.349557015870795,
|
|
"learning_rate": 9.84052137019499e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3807,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19790
|
|
},
|
|
{
|
|
"epoch": 0.737004708641194,
|
|
"grad_norm": 7.894890614614982,
|
|
"learning_rate": 9.814703688056319e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19800
|
|
},
|
|
{
|
|
"epoch": 0.737376933241518,
|
|
"grad_norm": 7.581638545978871,
|
|
"learning_rate": 9.78891164380343e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3875,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19810
|
|
},
|
|
{
|
|
"epoch": 0.7377491578418418,
|
|
"grad_norm": 6.878526707565156,
|
|
"learning_rate": 9.763145280981974e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3981,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19820
|
|
},
|
|
{
|
|
"epoch": 0.7381213824421656,
|
|
"grad_norm": 7.364132518416045,
|
|
"learning_rate": 9.737404643094235e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3712,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 19830
|
|
},
|
|
{
|
|
"epoch": 0.7384936070424895,
|
|
"grad_norm": 6.8485269462535685,
|
|
"learning_rate": 9.711689773599067e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3643,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19840
|
|
},
|
|
{
|
|
"epoch": 0.7388658316428133,
|
|
"grad_norm": 7.637506931195872,
|
|
"learning_rate": 9.686000715911819e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19850
|
|
},
|
|
{
|
|
"epoch": 0.7392380562431371,
|
|
"grad_norm": 9.15954565093133,
|
|
"learning_rate": 9.660337513404243e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 19860
|
|
},
|
|
{
|
|
"epoch": 0.7396102808434609,
|
|
"grad_norm": 8.10182338784461,
|
|
"learning_rate": 9.634700209404464e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3886,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 19870
|
|
},
|
|
{
|
|
"epoch": 0.7399825054437847,
|
|
"grad_norm": 8.11740456452422,
|
|
"learning_rate": 9.609088847196866e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3771,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19880
|
|
},
|
|
{
|
|
"epoch": 0.7403547300441086,
|
|
"grad_norm": 7.974441044144487,
|
|
"learning_rate": 9.583503470022053e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19890
|
|
},
|
|
{
|
|
"epoch": 0.7407269546444325,
|
|
"grad_norm": 5.850358126361301,
|
|
"learning_rate": 9.557944121076724e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3475,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 19900
|
|
},
|
|
{
|
|
"epoch": 0.7410991792447563,
|
|
"grad_norm": 7.604130038555371,
|
|
"learning_rate": 9.532410843513666e-08,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3718,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19910
|
|
},
|
|
{
|
|
"epoch": 0.7414714038450801,
|
|
"grad_norm": 7.607516257913704,
|
|
"learning_rate": 9.50690368044163e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.389,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19920
|
|
},
|
|
{
|
|
"epoch": 0.741843628445404,
|
|
"grad_norm": 7.351552308958112,
|
|
"learning_rate": 9.481422674925288e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19930
|
|
},
|
|
{
|
|
"epoch": 0.7422158530457278,
|
|
"grad_norm": 8.532648271464003,
|
|
"learning_rate": 9.455967869985151e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19940
|
|
},
|
|
{
|
|
"epoch": 0.7425880776460516,
|
|
"grad_norm": 8.151073949522868,
|
|
"learning_rate": 9.430539308597474e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.398,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 19950
|
|
},
|
|
{
|
|
"epoch": 0.7429603022463754,
|
|
"grad_norm": 6.778799302460495,
|
|
"learning_rate": 9.405137033694222e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3775,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 19960
|
|
},
|
|
{
|
|
"epoch": 0.7433325268466993,
|
|
"grad_norm": 8.055722261124101,
|
|
"learning_rate": 9.37976108816298e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3665,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 19970
|
|
},
|
|
{
|
|
"epoch": 0.7437047514470231,
|
|
"grad_norm": 8.076409667960185,
|
|
"learning_rate": 9.354411514846882e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.388,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.375,
|
|
"step": 19980
|
|
},
|
|
{
|
|
"epoch": 0.744076976047347,
|
|
"grad_norm": 8.155044125569761,
|
|
"learning_rate": 9.329088356544518e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3702,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 19990
|
|
},
|
|
{
|
|
"epoch": 0.7444492006476708,
|
|
"grad_norm": 7.062430072912223,
|
|
"learning_rate": 9.303791656009896e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.4022,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 20000
|
|
},
|
|
{
|
|
"epoch": 0.7444492006476708,
|
|
"eval_logits/chosen": -3.84375,
|
|
"eval_logits/rejected": -3.65625,
|
|
"eval_logps/chosen": -980.0,
|
|
"eval_logps/rejected": -1104.0,
|
|
"eval_loss": 0.4609435498714447,
|
|
"eval_rewards/accuracies": 0.7541024684906006,
|
|
"eval_rewards/chosen": -7.78125,
|
|
"eval_rewards/margins": 1.515625,
|
|
"eval_rewards/rejected": -9.3125,
|
|
"eval_runtime": 6315.556,
|
|
"eval_samples_per_second": 30.252,
|
|
"eval_steps_per_second": 0.473,
|
|
"step": 20000
|
|
},
|
|
{
|
|
"epoch": 0.7448214252479947,
|
|
"grad_norm": 7.229374720568194,
|
|
"learning_rate": 9.278521455952354e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20010
|
|
},
|
|
{
|
|
"epoch": 0.7451936498483185,
|
|
"grad_norm": 8.167124186852552,
|
|
"learning_rate": 9.253277799036488e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3916,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20020
|
|
},
|
|
{
|
|
"epoch": 0.7455658744486423,
|
|
"grad_norm": 8.465064003876963,
|
|
"learning_rate": 9.228060727882083e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20030
|
|
},
|
|
{
|
|
"epoch": 0.7459380990489661,
|
|
"grad_norm": 7.704146711257308,
|
|
"learning_rate": 9.202870285064021e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3782,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20040
|
|
},
|
|
{
|
|
"epoch": 0.74631032364929,
|
|
"grad_norm": 7.799209924133462,
|
|
"learning_rate": 9.177706513112246e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20050
|
|
},
|
|
{
|
|
"epoch": 0.7466825482496138,
|
|
"grad_norm": 8.614718312645907,
|
|
"learning_rate": 9.152569454511663e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3955,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 20060
|
|
},
|
|
{
|
|
"epoch": 0.7470547728499376,
|
|
"grad_norm": 7.7897872634298055,
|
|
"learning_rate": 9.12745915170209e-08,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3898,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20070
|
|
},
|
|
{
|
|
"epoch": 0.7474269974502615,
|
|
"grad_norm": 7.4682068389725,
|
|
"learning_rate": 9.102375647078145e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20080
|
|
},
|
|
{
|
|
"epoch": 0.7477992220505854,
|
|
"grad_norm": 8.58113372048121,
|
|
"learning_rate": 9.077318982989222e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -892.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3602,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.125,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20090
|
|
},
|
|
{
|
|
"epoch": 0.7481714466509092,
|
|
"grad_norm": 7.781552923204151,
|
|
"learning_rate": 9.052289201739397e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20100
|
|
},
|
|
{
|
|
"epoch": 0.748543671251233,
|
|
"grad_norm": 10.438152503948865,
|
|
"learning_rate": 9.027286345587354e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3837,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20110
|
|
},
|
|
{
|
|
"epoch": 0.7489158958515568,
|
|
"grad_norm": 7.327343616371855,
|
|
"learning_rate": 9.002310456746335e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 20120
|
|
},
|
|
{
|
|
"epoch": 0.7492881204518806,
|
|
"grad_norm": 7.92873754579415,
|
|
"learning_rate": 8.977361577384013e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3707,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20130
|
|
},
|
|
{
|
|
"epoch": 0.7496603450522045,
|
|
"grad_norm": 8.505563262448055,
|
|
"learning_rate": 8.952439749622497e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20140
|
|
},
|
|
{
|
|
"epoch": 0.7500325696525283,
|
|
"grad_norm": 9.172356645709966,
|
|
"learning_rate": 8.927545015538212e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.395,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20150
|
|
},
|
|
{
|
|
"epoch": 0.7504047942528522,
|
|
"grad_norm": 8.039324088219162,
|
|
"learning_rate": 8.902677417161839e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3796,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20160
|
|
},
|
|
{
|
|
"epoch": 0.750777018853176,
|
|
"grad_norm": 7.604124822006528,
|
|
"learning_rate": 8.877836996478252e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3918,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 20170
|
|
},
|
|
{
|
|
"epoch": 0.7511492434534999,
|
|
"grad_norm": 8.77042520233627,
|
|
"learning_rate": 8.853023795426421e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3925,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20180
|
|
},
|
|
{
|
|
"epoch": 0.7515214680538237,
|
|
"grad_norm": 9.282189867454193,
|
|
"learning_rate": 8.828237855899373e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.379,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20190
|
|
},
|
|
{
|
|
"epoch": 0.7518936926541475,
|
|
"grad_norm": 9.062662745199882,
|
|
"learning_rate": 8.803479219744111e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3808,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20200
|
|
},
|
|
{
|
|
"epoch": 0.7522659172544713,
|
|
"grad_norm": 9.16925559437944,
|
|
"learning_rate": 8.778747928761548e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3764,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20210
|
|
},
|
|
{
|
|
"epoch": 0.7526381418547952,
|
|
"grad_norm": 6.655122647231586,
|
|
"learning_rate": 8.7540440247064e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3738,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20220
|
|
},
|
|
{
|
|
"epoch": 0.753010366455119,
|
|
"grad_norm": 7.644796701528563,
|
|
"learning_rate": 8.729367549287168e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20230
|
|
},
|
|
{
|
|
"epoch": 0.7533825910554428,
|
|
"grad_norm": 8.027468922579871,
|
|
"learning_rate": 8.704718544166046e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3834,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20240
|
|
},
|
|
{
|
|
"epoch": 0.7537548156557667,
|
|
"grad_norm": 7.026643472130103,
|
|
"learning_rate": 8.680097050958834e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20250
|
|
},
|
|
{
|
|
"epoch": 0.7541270402560906,
|
|
"grad_norm": 7.621802790316961,
|
|
"learning_rate": 8.655503111234905e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3866,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20260
|
|
},
|
|
{
|
|
"epoch": 0.7544992648564144,
|
|
"grad_norm": 6.1604333527819986,
|
|
"learning_rate": 8.630936766517081e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.381,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20270
|
|
},
|
|
{
|
|
"epoch": 0.7548714894567382,
|
|
"grad_norm": 8.923527822101214,
|
|
"learning_rate": 8.606398058281619e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3654,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20280
|
|
},
|
|
{
|
|
"epoch": 0.755243714057062,
|
|
"grad_norm": 8.063527021099807,
|
|
"learning_rate": 8.581887027958107e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3709,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20290
|
|
},
|
|
{
|
|
"epoch": 0.7556159386573859,
|
|
"grad_norm": 8.727394960807557,
|
|
"learning_rate": 8.557403716929418e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3826,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.5,
|
|
"step": 20300
|
|
},
|
|
{
|
|
"epoch": 0.7559881632577097,
|
|
"grad_norm": 8.725646995516813,
|
|
"learning_rate": 8.532948166531592e-08,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4088,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20310
|
|
},
|
|
{
|
|
"epoch": 0.7563603878580335,
|
|
"grad_norm": 9.805080230944224,
|
|
"learning_rate": 8.508520418053839e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3947,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20320
|
|
},
|
|
{
|
|
"epoch": 0.7567326124583573,
|
|
"grad_norm": 7.494381882010992,
|
|
"learning_rate": 8.484120512738386e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.3855,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -8.875,
|
|
"step": 20330
|
|
},
|
|
{
|
|
"epoch": 0.7571048370586813,
|
|
"grad_norm": 7.503765916756013,
|
|
"learning_rate": 8.459748491780502e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3819,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20340
|
|
},
|
|
{
|
|
"epoch": 0.7574770616590051,
|
|
"grad_norm": 7.877829541109141,
|
|
"learning_rate": 8.435404396328355e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3933,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20350
|
|
},
|
|
{
|
|
"epoch": 0.7578492862593289,
|
|
"grad_norm": 7.83578783898455,
|
|
"learning_rate": 8.411088267482944e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20360
|
|
},
|
|
{
|
|
"epoch": 0.7582215108596527,
|
|
"grad_norm": 8.369499853849895,
|
|
"learning_rate": 8.386800146298087e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20370
|
|
},
|
|
{
|
|
"epoch": 0.7585937354599765,
|
|
"grad_norm": 7.757113721217556,
|
|
"learning_rate": 8.362540073780275e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20380
|
|
},
|
|
{
|
|
"epoch": 0.7589659600603004,
|
|
"grad_norm": 7.2147770108258475,
|
|
"learning_rate": 8.338308090888696e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20390
|
|
},
|
|
{
|
|
"epoch": 0.7593381846606242,
|
|
"grad_norm": 7.937339755562026,
|
|
"learning_rate": 8.314104238535066e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4055,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20400
|
|
},
|
|
{
|
|
"epoch": 0.759710409260948,
|
|
"grad_norm": 7.304856177449764,
|
|
"learning_rate": 8.289928557583639e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3647,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20410
|
|
},
|
|
{
|
|
"epoch": 0.7600826338612718,
|
|
"grad_norm": 6.946658387011915,
|
|
"learning_rate": 8.265781088851068e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.377,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20420
|
|
},
|
|
{
|
|
"epoch": 0.7604548584615958,
|
|
"grad_norm": 7.656025627750178,
|
|
"learning_rate": 8.241661873106426e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4053,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20430
|
|
},
|
|
{
|
|
"epoch": 0.7608270830619196,
|
|
"grad_norm": 7.779021019839977,
|
|
"learning_rate": 8.217570951071055e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4056,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20440
|
|
},
|
|
{
|
|
"epoch": 0.7611993076622434,
|
|
"grad_norm": 8.090365384432998,
|
|
"learning_rate": 8.193508363418514e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3717,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20450
|
|
},
|
|
{
|
|
"epoch": 0.7615715322625672,
|
|
"grad_norm": 7.740825702154877,
|
|
"learning_rate": 8.169474150774563e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4059,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20460
|
|
},
|
|
{
|
|
"epoch": 0.7619437568628911,
|
|
"grad_norm": 8.0282151613539,
|
|
"learning_rate": 8.145468353717006e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3773,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20470
|
|
},
|
|
{
|
|
"epoch": 0.7623159814632149,
|
|
"grad_norm": 7.109356084948688,
|
|
"learning_rate": 8.121491012775733e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20480
|
|
},
|
|
{
|
|
"epoch": 0.7626882060635387,
|
|
"grad_norm": 7.432573165619803,
|
|
"learning_rate": 8.09754216843253e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3708,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20490
|
|
},
|
|
{
|
|
"epoch": 0.7630604306638625,
|
|
"grad_norm": 8.17756765601075,
|
|
"learning_rate": 8.073621861121113e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3634,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20500
|
|
},
|
|
{
|
|
"epoch": 0.7634326552641864,
|
|
"grad_norm": 7.531461301284386,
|
|
"learning_rate": 8.049730131227005e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20510
|
|
},
|
|
{
|
|
"epoch": 0.7638048798645103,
|
|
"grad_norm": 8.00621845893518,
|
|
"learning_rate": 8.02586701908746e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3994,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 20520
|
|
},
|
|
{
|
|
"epoch": 0.7641771044648341,
|
|
"grad_norm": 8.65814652578834,
|
|
"learning_rate": 8.002032564991459e-08,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3981,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.96875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 20530
|
|
},
|
|
{
|
|
"epoch": 0.7645493290651579,
|
|
"grad_norm": 7.925794216385904,
|
|
"learning_rate": 7.978226809179558e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3655,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20540
|
|
},
|
|
{
|
|
"epoch": 0.7649215536654818,
|
|
"grad_norm": 8.818847584338986,
|
|
"learning_rate": 7.95444979184389e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4009,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20550
|
|
},
|
|
{
|
|
"epoch": 0.7652937782658056,
|
|
"grad_norm": 8.531576775880923,
|
|
"learning_rate": 7.930701553128036e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20560
|
|
},
|
|
{
|
|
"epoch": 0.7656660028661294,
|
|
"grad_norm": 7.441988939668615,
|
|
"learning_rate": 7.906982133127018e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3772,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20570
|
|
},
|
|
{
|
|
"epoch": 0.7660382274664532,
|
|
"grad_norm": 8.49191277076841,
|
|
"learning_rate": 7.883291571887196e-08,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3892,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20580
|
|
},
|
|
{
|
|
"epoch": 0.766410452066777,
|
|
"grad_norm": 7.647922913744324,
|
|
"learning_rate": 7.859629909406196e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20590
|
|
},
|
|
{
|
|
"epoch": 0.766782676667101,
|
|
"grad_norm": 9.018546993615995,
|
|
"learning_rate": 7.83599718563287e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20600
|
|
},
|
|
{
|
|
"epoch": 0.7671549012674248,
|
|
"grad_norm": 8.529568634897798,
|
|
"learning_rate": 7.812393440467177e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20610
|
|
},
|
|
{
|
|
"epoch": 0.7675271258677486,
|
|
"grad_norm": 7.151111450635194,
|
|
"learning_rate": 7.78881871376021e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3737,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20620
|
|
},
|
|
{
|
|
"epoch": 0.7678993504680725,
|
|
"grad_norm": 7.072977264718187,
|
|
"learning_rate": 7.765273045314005e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3811,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 20630
|
|
},
|
|
{
|
|
"epoch": 0.7682715750683963,
|
|
"grad_norm": 6.857862571116441,
|
|
"learning_rate": 7.741756474881588e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20640
|
|
},
|
|
{
|
|
"epoch": 0.7686437996687201,
|
|
"grad_norm": 8.916559656793487,
|
|
"learning_rate": 7.718269042166817e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3756,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20650
|
|
},
|
|
{
|
|
"epoch": 0.7690160242690439,
|
|
"grad_norm": 7.098516540124456,
|
|
"learning_rate": 7.694810786824388e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3985,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20660
|
|
},
|
|
{
|
|
"epoch": 0.7693882488693677,
|
|
"grad_norm": 7.495275256911239,
|
|
"learning_rate": 7.671381748459715e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20670
|
|
},
|
|
{
|
|
"epoch": 0.7697604734696916,
|
|
"grad_norm": 7.272179232164947,
|
|
"learning_rate": 7.647981966628899e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -888.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.362,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.21875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20680
|
|
},
|
|
{
|
|
"epoch": 0.7701326980700155,
|
|
"grad_norm": 7.8293967127420565,
|
|
"learning_rate": 7.624611480838647e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3641,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20690
|
|
},
|
|
{
|
|
"epoch": 0.7705049226703393,
|
|
"grad_norm": 8.239059377348045,
|
|
"learning_rate": 7.601270330546177e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3777,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20700
|
|
},
|
|
{
|
|
"epoch": 0.7708771472706631,
|
|
"grad_norm": 7.133903881225811,
|
|
"learning_rate": 7.577958555159209e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3671,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20710
|
|
},
|
|
{
|
|
"epoch": 0.771249371870987,
|
|
"grad_norm": 7.712855003298118,
|
|
"learning_rate": 7.554676194035859e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 20720
|
|
},
|
|
{
|
|
"epoch": 0.7716215964713108,
|
|
"grad_norm": 8.146410654868244,
|
|
"learning_rate": 7.531423286484584e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.386,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 20730
|
|
},
|
|
{
|
|
"epoch": 0.7719938210716346,
|
|
"grad_norm": 8.548245220453232,
|
|
"learning_rate": 7.508199871764098e-08,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3768,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20740
|
|
},
|
|
{
|
|
"epoch": 0.7723660456719584,
|
|
"grad_norm": 8.647953173795505,
|
|
"learning_rate": 7.48500598908334e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3733,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20750
|
|
},
|
|
{
|
|
"epoch": 0.7727382702722823,
|
|
"grad_norm": 7.6185301826808205,
|
|
"learning_rate": 7.46184167760138e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -9.0,
|
|
"step": 20760
|
|
},
|
|
{
|
|
"epoch": 0.7731104948726061,
|
|
"grad_norm": 6.6226862305042715,
|
|
"learning_rate": 7.43870697642737e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3856,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 20770
|
|
},
|
|
{
|
|
"epoch": 0.77348271947293,
|
|
"grad_norm": 8.132461241837545,
|
|
"learning_rate": 7.415601924620465e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3844,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20780
|
|
},
|
|
{
|
|
"epoch": 0.7738549440732538,
|
|
"grad_norm": 7.192402985065547,
|
|
"learning_rate": 7.392526561189752e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3362,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20790
|
|
},
|
|
{
|
|
"epoch": 0.7742271686735777,
|
|
"grad_norm": 8.077422877583272,
|
|
"learning_rate": 7.369480925094204e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 20800
|
|
},
|
|
{
|
|
"epoch": 0.7745993932739015,
|
|
"grad_norm": 7.57724007411771,
|
|
"learning_rate": 7.346465055242606e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 20810
|
|
},
|
|
{
|
|
"epoch": 0.7749716178742253,
|
|
"grad_norm": 9.919604337829874,
|
|
"learning_rate": 7.323478990493495e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4033,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20820
|
|
},
|
|
{
|
|
"epoch": 0.7753438424745491,
|
|
"grad_norm": 7.76234008263285,
|
|
"learning_rate": 7.30052276965506e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3936,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20830
|
|
},
|
|
{
|
|
"epoch": 0.775716067074873,
|
|
"grad_norm": 7.808790117506864,
|
|
"learning_rate": 7.277596431485126e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3829,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20840
|
|
},
|
|
{
|
|
"epoch": 0.7760882916751968,
|
|
"grad_norm": 8.237691531471603,
|
|
"learning_rate": 7.254700014691059e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 20850
|
|
},
|
|
{
|
|
"epoch": 0.7764605162755206,
|
|
"grad_norm": 7.595722083664976,
|
|
"learning_rate": 7.231833557929712e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3857,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 20860
|
|
},
|
|
{
|
|
"epoch": 0.7768327408758445,
|
|
"grad_norm": 7.867918411032424,
|
|
"learning_rate": 7.208997099807356e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3883,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20870
|
|
},
|
|
{
|
|
"epoch": 0.7772049654761684,
|
|
"grad_norm": 7.232988532765224,
|
|
"learning_rate": 7.186190678879603e-08,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.484375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20880
|
|
},
|
|
{
|
|
"epoch": 0.7775771900764922,
|
|
"grad_norm": 8.068818601577966,
|
|
"learning_rate": 7.163414333651357e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20890
|
|
},
|
|
{
|
|
"epoch": 0.777949414676816,
|
|
"grad_norm": 7.920032032922766,
|
|
"learning_rate": 7.140668102576756e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4003,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 20900
|
|
},
|
|
{
|
|
"epoch": 0.7783216392771398,
|
|
"grad_norm": 9.290574374409536,
|
|
"learning_rate": 7.117952024059084e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3967,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20910
|
|
},
|
|
{
|
|
"epoch": 0.7786938638774636,
|
|
"grad_norm": 8.958823564272384,
|
|
"learning_rate": 7.095266136450714e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 20920
|
|
},
|
|
{
|
|
"epoch": 0.7790660884777875,
|
|
"grad_norm": 6.54899835417583,
|
|
"learning_rate": 7.072610478053051e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20930
|
|
},
|
|
{
|
|
"epoch": 0.7794383130781113,
|
|
"grad_norm": 7.35832162223178,
|
|
"learning_rate": 7.049985087116469e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20940
|
|
},
|
|
{
|
|
"epoch": 0.7798105376784352,
|
|
"grad_norm": 7.795520689142863,
|
|
"learning_rate": 7.02739000184023e-08,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20950
|
|
},
|
|
{
|
|
"epoch": 0.780182762278759,
|
|
"grad_norm": 7.119655645823678,
|
|
"learning_rate": 7.004825260372449e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20960
|
|
},
|
|
{
|
|
"epoch": 0.7805549868790829,
|
|
"grad_norm": 6.933061882953764,
|
|
"learning_rate": 6.982290900809976e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 20970
|
|
},
|
|
{
|
|
"epoch": 0.7809272114794067,
|
|
"grad_norm": 9.12309279591005,
|
|
"learning_rate": 6.959786961198396e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4159,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 20980
|
|
},
|
|
{
|
|
"epoch": 0.7812994360797305,
|
|
"grad_norm": 6.7930224891679565,
|
|
"learning_rate": 6.937313479531925e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3714,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 20990
|
|
},
|
|
{
|
|
"epoch": 0.7816716606800543,
|
|
"grad_norm": 5.9913201128461155,
|
|
"learning_rate": 6.914870493753363e-08,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -900.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3666,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 21000
|
|
},
|
|
{
|
|
"epoch": 0.7820438852803782,
|
|
"grad_norm": 6.681323513292506,
|
|
"learning_rate": 6.892458041754001e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3727,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21010
|
|
},
|
|
{
|
|
"epoch": 0.782416109880702,
|
|
"grad_norm": 9.098672483575903,
|
|
"learning_rate": 6.870076161373603e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4061,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.359375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21020
|
|
},
|
|
{
|
|
"epoch": 0.7827883344810258,
|
|
"grad_norm": 7.343110287716684,
|
|
"learning_rate": 6.847724890400306e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3946,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21030
|
|
},
|
|
{
|
|
"epoch": 0.7831605590813497,
|
|
"grad_norm": 7.652151391778288,
|
|
"learning_rate": 6.825404266570572e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3908,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21040
|
|
},
|
|
{
|
|
"epoch": 0.7835327836816736,
|
|
"grad_norm": 8.13363484799162,
|
|
"learning_rate": 6.803114327569126e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.391,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 21050
|
|
},
|
|
{
|
|
"epoch": 0.7839050082819974,
|
|
"grad_norm": 9.676732138789387,
|
|
"learning_rate": 6.780855111028863e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21060
|
|
},
|
|
{
|
|
"epoch": 0.7842772328823212,
|
|
"grad_norm": 8.47031439643481,
|
|
"learning_rate": 6.75862665453083e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21070
|
|
},
|
|
{
|
|
"epoch": 0.784649457482645,
|
|
"grad_norm": 7.036929041425403,
|
|
"learning_rate": 6.736428995604143e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21080
|
|
},
|
|
{
|
|
"epoch": 0.7850216820829689,
|
|
"grad_norm": 9.208249944290017,
|
|
"learning_rate": 6.714262171725904e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3939,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 21090
|
|
},
|
|
{
|
|
"epoch": 0.7853939066832927,
|
|
"grad_norm": 8.484606926876186,
|
|
"learning_rate": 6.692126220321181e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3917,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21100
|
|
},
|
|
{
|
|
"epoch": 0.7857661312836165,
|
|
"grad_norm": 7.125471846455673,
|
|
"learning_rate": 6.67002117876288e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4049,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21110
|
|
},
|
|
{
|
|
"epoch": 0.7861383558839403,
|
|
"grad_norm": 5.880083182703317,
|
|
"learning_rate": 6.647947084371755e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3726,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21120
|
|
},
|
|
{
|
|
"epoch": 0.7865105804842643,
|
|
"grad_norm": 8.487687110153544,
|
|
"learning_rate": 6.6259039744163e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3769,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21130
|
|
},
|
|
{
|
|
"epoch": 0.7868828050845881,
|
|
"grad_norm": 7.493554933062326,
|
|
"learning_rate": 6.6038918861127e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3867,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21140
|
|
},
|
|
{
|
|
"epoch": 0.7872550296849119,
|
|
"grad_norm": 7.787326827062894,
|
|
"learning_rate": 6.581910856624748e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3839,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21150
|
|
},
|
|
{
|
|
"epoch": 0.7876272542852357,
|
|
"grad_norm": 6.862249440691552,
|
|
"learning_rate": 6.559960923063829e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3579,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21160
|
|
},
|
|
{
|
|
"epoch": 0.7879994788855595,
|
|
"grad_norm": 7.108745671917929,
|
|
"learning_rate": 6.538042122488786e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21170
|
|
},
|
|
{
|
|
"epoch": 0.7883717034858834,
|
|
"grad_norm": 8.76879143819679,
|
|
"learning_rate": 6.516154491905943e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.374,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21180
|
|
},
|
|
{
|
|
"epoch": 0.7887439280862072,
|
|
"grad_norm": 9.41011641425454,
|
|
"learning_rate": 6.494298068268986e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3854,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21190
|
|
},
|
|
{
|
|
"epoch": 0.789116152686531,
|
|
"grad_norm": 6.867937454601424,
|
|
"learning_rate": 6.472472888478888e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3659,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21200
|
|
},
|
|
{
|
|
"epoch": 0.7894883772868548,
|
|
"grad_norm": 7.898279411494632,
|
|
"learning_rate": 6.450678989383901e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3801,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21210
|
|
},
|
|
{
|
|
"epoch": 0.7898606018871788,
|
|
"grad_norm": 8.349088586737386,
|
|
"learning_rate": 6.428916407779452e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.396,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21220
|
|
},
|
|
{
|
|
"epoch": 0.7902328264875026,
|
|
"grad_norm": 7.178604371113771,
|
|
"learning_rate": 6.407185180408104e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3735,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21230
|
|
},
|
|
{
|
|
"epoch": 0.7906050510878264,
|
|
"grad_norm": 7.971353705858471,
|
|
"learning_rate": 6.385485343959457e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21240
|
|
},
|
|
{
|
|
"epoch": 0.7909772756881502,
|
|
"grad_norm": 7.764996162314372,
|
|
"learning_rate": 6.363816935070152e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3729,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21250
|
|
},
|
|
{
|
|
"epoch": 0.7913495002884741,
|
|
"grad_norm": 8.554923321553373,
|
|
"learning_rate": 6.34217999032372e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3649,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21260
|
|
},
|
|
{
|
|
"epoch": 0.7917217248887979,
|
|
"grad_norm": 7.765171807380977,
|
|
"learning_rate": 6.320574546250627e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4074,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21270
|
|
},
|
|
{
|
|
"epoch": 0.7920939494891217,
|
|
"grad_norm": 7.448869667181042,
|
|
"learning_rate": 6.29900063932812e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21280
|
|
},
|
|
{
|
|
"epoch": 0.7924661740894455,
|
|
"grad_norm": 7.414353312143225,
|
|
"learning_rate": 6.277458305980198e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21290
|
|
},
|
|
{
|
|
"epoch": 0.7928383986897694,
|
|
"grad_norm": 8.073611511568327,
|
|
"learning_rate": 6.255947582577576e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3738,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21300
|
|
},
|
|
{
|
|
"epoch": 0.7932106232900933,
|
|
"grad_norm": 7.891953755780748,
|
|
"learning_rate": 6.234468505437565e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21310
|
|
},
|
|
{
|
|
"epoch": 0.7935828478904171,
|
|
"grad_norm": 7.8617340119026435,
|
|
"learning_rate": 6.213021110824099e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.366,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21320
|
|
},
|
|
{
|
|
"epoch": 0.7939550724907409,
|
|
"grad_norm": 8.622210695720963,
|
|
"learning_rate": 6.191605434947566e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3603,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21330
|
|
},
|
|
{
|
|
"epoch": 0.7943272970910648,
|
|
"grad_norm": 8.822469238560753,
|
|
"learning_rate": 6.170221513964841e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3918,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.4453125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 21340
|
|
},
|
|
{
|
|
"epoch": 0.7946995216913886,
|
|
"grad_norm": 9.477480954696437,
|
|
"learning_rate": 6.148869383979172e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21350
|
|
},
|
|
{
|
|
"epoch": 0.7950717462917124,
|
|
"grad_norm": 7.800684410898062,
|
|
"learning_rate": 6.127549081040117e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.3906,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 21360
|
|
},
|
|
{
|
|
"epoch": 0.7954439708920362,
|
|
"grad_norm": 7.102879842820928,
|
|
"learning_rate": 6.106260641143546e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3767,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21370
|
|
},
|
|
{
|
|
"epoch": 0.79581619549236,
|
|
"grad_norm": 7.483038976361463,
|
|
"learning_rate": 6.085004100231475e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3644,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21380
|
|
},
|
|
{
|
|
"epoch": 0.796188420092684,
|
|
"grad_norm": 6.6427991482519495,
|
|
"learning_rate": 6.063779494192112e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21390
|
|
},
|
|
{
|
|
"epoch": 0.7965606446930078,
|
|
"grad_norm": 6.772847514249572,
|
|
"learning_rate": 6.042586858859703e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4042,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21400
|
|
},
|
|
{
|
|
"epoch": 0.7969328692933316,
|
|
"grad_norm": 8.380996682054466,
|
|
"learning_rate": 6.021426230014572e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3633,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21410
|
|
},
|
|
{
|
|
"epoch": 0.7973050938936554,
|
|
"grad_norm": 7.393061290489446,
|
|
"learning_rate": 6.000297643382957e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3583,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21420
|
|
},
|
|
{
|
|
"epoch": 0.7976773184939793,
|
|
"grad_norm": 7.596972068499439,
|
|
"learning_rate": 5.979201134637015e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3895,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21430
|
|
},
|
|
{
|
|
"epoch": 0.7980495430943031,
|
|
"grad_norm": 8.606395697851516,
|
|
"learning_rate": 5.9581367393947616e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3935,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.0625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21440
|
|
},
|
|
{
|
|
"epoch": 0.7984217676946269,
|
|
"grad_norm": 8.576325852981057,
|
|
"learning_rate": 5.937104493219952e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3683,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21450
|
|
},
|
|
{
|
|
"epoch": 0.7987939922949507,
|
|
"grad_norm": 9.044220330197067,
|
|
"learning_rate": 5.916104431622121e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21460
|
|
},
|
|
{
|
|
"epoch": 0.7991662168952746,
|
|
"grad_norm": 8.056248439052354,
|
|
"learning_rate": 5.895136590056407e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21470
|
|
},
|
|
{
|
|
"epoch": 0.7995384414955985,
|
|
"grad_norm": 7.3487690781896156,
|
|
"learning_rate": 5.8742010039235957e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3815,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21480
|
|
},
|
|
{
|
|
"epoch": 0.7999106660959223,
|
|
"grad_norm": 7.263409193751304,
|
|
"learning_rate": 5.853297708569979e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3749,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21490
|
|
},
|
|
{
|
|
"epoch": 0.8002828906962461,
|
|
"grad_norm": 8.17161896821374,
|
|
"learning_rate": 5.832426739287355e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3785,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21500
|
|
},
|
|
{
|
|
"epoch": 0.80065511529657,
|
|
"grad_norm": 7.348224583076228,
|
|
"learning_rate": 5.811588131312936e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3703,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21510
|
|
},
|
|
{
|
|
"epoch": 0.8010273398968938,
|
|
"grad_norm": 7.8515106130017465,
|
|
"learning_rate": 5.790781919829299e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3882,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21520
|
|
},
|
|
{
|
|
"epoch": 0.8013995644972176,
|
|
"grad_norm": 8.431897441317817,
|
|
"learning_rate": 5.770008139964333e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.379,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21530
|
|
},
|
|
{
|
|
"epoch": 0.8017717890975414,
|
|
"grad_norm": 7.044283261772037,
|
|
"learning_rate": 5.749266826791146e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4051,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21540
|
|
},
|
|
{
|
|
"epoch": 0.8021440136978653,
|
|
"grad_norm": 7.207238405493094,
|
|
"learning_rate": 5.728558015328061e-08,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21550
|
|
},
|
|
{
|
|
"epoch": 0.8025162382981891,
|
|
"grad_norm": 6.7503227717180065,
|
|
"learning_rate": 5.707881740538509e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21560
|
|
},
|
|
{
|
|
"epoch": 0.802888462898513,
|
|
"grad_norm": 8.015239223758112,
|
|
"learning_rate": 5.687238037331002e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3865,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21570
|
|
},
|
|
{
|
|
"epoch": 0.8032606874988368,
|
|
"grad_norm": 7.53506828204368,
|
|
"learning_rate": 5.666626940559038e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3701,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21580
|
|
},
|
|
{
|
|
"epoch": 0.8036329120991607,
|
|
"grad_norm": 9.477937811018823,
|
|
"learning_rate": 5.646048485021085e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21590
|
|
},
|
|
{
|
|
"epoch": 0.8040051366994845,
|
|
"grad_norm": 7.241143736260145,
|
|
"learning_rate": 5.625502705460497e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3986,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 21600
|
|
},
|
|
{
|
|
"epoch": 0.8043773612998083,
|
|
"grad_norm": 8.316423810645324,
|
|
"learning_rate": 5.6049896365654577e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 21610
|
|
},
|
|
{
|
|
"epoch": 0.8047495859001321,
|
|
"grad_norm": 8.471348936219883,
|
|
"learning_rate": 5.584509312968927e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 21620
|
|
},
|
|
{
|
|
"epoch": 0.805121810500456,
|
|
"grad_norm": 8.488686594253211,
|
|
"learning_rate": 5.5640617692485654e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.368,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21630
|
|
},
|
|
{
|
|
"epoch": 0.8054940351007798,
|
|
"grad_norm": 8.022332025711552,
|
|
"learning_rate": 5.5436470399267103e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21640
|
|
},
|
|
{
|
|
"epoch": 0.8058662597011036,
|
|
"grad_norm": 8.385532913651012,
|
|
"learning_rate": 5.523265159470289e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3906,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21650
|
|
},
|
|
{
|
|
"epoch": 0.8062384843014275,
|
|
"grad_norm": 8.048136999776284,
|
|
"learning_rate": 5.5029161622907716e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21660
|
|
},
|
|
{
|
|
"epoch": 0.8066107089017513,
|
|
"grad_norm": 7.080272771339101,
|
|
"learning_rate": 5.482600082744096e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3613,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21670
|
|
},
|
|
{
|
|
"epoch": 0.8069829335020752,
|
|
"grad_norm": 8.022999105424757,
|
|
"learning_rate": 5.4623169551306445e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.376,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21680
|
|
},
|
|
{
|
|
"epoch": 0.807355158102399,
|
|
"grad_norm": 8.1891218856332,
|
|
"learning_rate": 5.442066813695151e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3838,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 21690
|
|
},
|
|
{
|
|
"epoch": 0.8077273827027228,
|
|
"grad_norm": 9.167749996669224,
|
|
"learning_rate": 5.421849692626665e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3844,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 21700
|
|
},
|
|
{
|
|
"epoch": 0.8080996073030466,
|
|
"grad_norm": 7.331126266861898,
|
|
"learning_rate": 5.401665626058491e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21710
|
|
},
|
|
{
|
|
"epoch": 0.8084718319033705,
|
|
"grad_norm": 7.498518355432903,
|
|
"learning_rate": 5.3815146480681056e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3868,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21720
|
|
},
|
|
{
|
|
"epoch": 0.8088440565036943,
|
|
"grad_norm": 8.283497897380839,
|
|
"learning_rate": 5.361396792677142e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3714,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21730
|
|
},
|
|
{
|
|
"epoch": 0.8092162811040181,
|
|
"grad_norm": 7.063767516232003,
|
|
"learning_rate": 5.341312093851305e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21740
|
|
},
|
|
{
|
|
"epoch": 0.809588505704342,
|
|
"grad_norm": 9.286242725028764,
|
|
"learning_rate": 5.3212605855003254e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.399,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21750
|
|
},
|
|
{
|
|
"epoch": 0.8099607303046659,
|
|
"grad_norm": 8.040002128923751,
|
|
"learning_rate": 5.301242301477879e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21760
|
|
},
|
|
{
|
|
"epoch": 0.8103329549049897,
|
|
"grad_norm": 7.83773562835892,
|
|
"learning_rate": 5.281257275581563e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3991,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21770
|
|
},
|
|
{
|
|
"epoch": 0.8107051795053135,
|
|
"grad_norm": 7.457966772154983,
|
|
"learning_rate": 5.261305541552824e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21780
|
|
},
|
|
{
|
|
"epoch": 0.8110774041056373,
|
|
"grad_norm": 8.880566736725283,
|
|
"learning_rate": 5.241387133076894e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3856,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21790
|
|
},
|
|
{
|
|
"epoch": 0.8114496287059612,
|
|
"grad_norm": 8.442215455062588,
|
|
"learning_rate": 5.2215020837827527e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3646,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21800
|
|
},
|
|
{
|
|
"epoch": 0.811821853306285,
|
|
"grad_norm": 7.190054133862455,
|
|
"learning_rate": 5.201650427243034e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3872,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21810
|
|
},
|
|
{
|
|
"epoch": 0.8121940779066088,
|
|
"grad_norm": 8.45045412347461,
|
|
"learning_rate": 5.181832196974015e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3925,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21820
|
|
},
|
|
{
|
|
"epoch": 0.8125663025069327,
|
|
"grad_norm": 7.592159914100721,
|
|
"learning_rate": 5.162047426435537e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3731,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21830
|
|
},
|
|
{
|
|
"epoch": 0.8129385271072566,
|
|
"grad_norm": 7.8832989786653895,
|
|
"learning_rate": 5.142296149030945e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3893,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21840
|
|
},
|
|
{
|
|
"epoch": 0.8133107517075804,
|
|
"grad_norm": 8.96036832406191,
|
|
"learning_rate": 5.1225783981070245e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3555,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 21850
|
|
},
|
|
{
|
|
"epoch": 0.8136829763079042,
|
|
"grad_norm": 8.109260781631969,
|
|
"learning_rate": 5.102894206953975e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.364,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21860
|
|
},
|
|
{
|
|
"epoch": 0.814055200908228,
|
|
"grad_norm": 8.53910796216198,
|
|
"learning_rate": 5.083243608805332e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3846,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -9.125,
|
|
"step": 21870
|
|
},
|
|
{
|
|
"epoch": 0.8144274255085518,
|
|
"grad_norm": 8.254601607957722,
|
|
"learning_rate": 5.0636266368379085e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3997,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21880
|
|
},
|
|
{
|
|
"epoch": 0.8147996501088757,
|
|
"grad_norm": 7.659198053205626,
|
|
"learning_rate": 5.0440433241717596e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21890
|
|
},
|
|
{
|
|
"epoch": 0.8151718747091995,
|
|
"grad_norm": 9.295196673430175,
|
|
"learning_rate": 5.024493703870084e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21900
|
|
},
|
|
{
|
|
"epoch": 0.8155440993095233,
|
|
"grad_norm": 8.50101879671381,
|
|
"learning_rate": 5.004977808939223e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.406,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21910
|
|
},
|
|
{
|
|
"epoch": 0.8159163239098473,
|
|
"grad_norm": 7.550779060901819,
|
|
"learning_rate": 4.9854956723285687e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3709,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 21920
|
|
},
|
|
{
|
|
"epoch": 0.8162885485101711,
|
|
"grad_norm": 7.417192722184543,
|
|
"learning_rate": 4.966047326930517e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3755,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 21930
|
|
},
|
|
{
|
|
"epoch": 0.8166607731104949,
|
|
"grad_norm": 7.222615677137182,
|
|
"learning_rate": 4.94663280558042e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.384,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 21940
|
|
},
|
|
{
|
|
"epoch": 0.8170329977108187,
|
|
"grad_norm": 7.131498824701582,
|
|
"learning_rate": 4.9272521410565065e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 21950
|
|
},
|
|
{
|
|
"epoch": 0.8174052223111425,
|
|
"grad_norm": 8.403158445847188,
|
|
"learning_rate": 4.907905366079859e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4051,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 21960
|
|
},
|
|
{
|
|
"epoch": 0.8177774469114664,
|
|
"grad_norm": 8.528098291867307,
|
|
"learning_rate": 4.888592513314338e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 21970
|
|
},
|
|
{
|
|
"epoch": 0.8181496715117902,
|
|
"grad_norm": 7.27897857682044,
|
|
"learning_rate": 4.869313615366541e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4109,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 21980
|
|
},
|
|
{
|
|
"epoch": 0.818521896112114,
|
|
"grad_norm": 7.884182272367873,
|
|
"learning_rate": 4.8500687047857145e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4057,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 21990
|
|
},
|
|
{
|
|
"epoch": 0.8188941207124378,
|
|
"grad_norm": 8.0330694841123,
|
|
"learning_rate": 4.830857814063744e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3945,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22000
|
|
},
|
|
{
|
|
"epoch": 0.8192663453127618,
|
|
"grad_norm": 8.50141414960529,
|
|
"learning_rate": 4.8116809756350767e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 22010
|
|
},
|
|
{
|
|
"epoch": 0.8196385699130856,
|
|
"grad_norm": 7.171734973716553,
|
|
"learning_rate": 4.7925382218766604e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22020
|
|
},
|
|
{
|
|
"epoch": 0.8200107945134094,
|
|
"grad_norm": 7.046688030770942,
|
|
"learning_rate": 4.7734295851079046e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.392,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22030
|
|
},
|
|
{
|
|
"epoch": 0.8203830191137332,
|
|
"grad_norm": 7.983163473147121,
|
|
"learning_rate": 4.7543550975906013e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 22040
|
|
},
|
|
{
|
|
"epoch": 0.8207552437140571,
|
|
"grad_norm": 7.806722010189656,
|
|
"learning_rate": 4.735314791528908e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3847,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22050
|
|
},
|
|
{
|
|
"epoch": 0.8211274683143809,
|
|
"grad_norm": 7.299482939240331,
|
|
"learning_rate": 4.716308699069257e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22060
|
|
},
|
|
{
|
|
"epoch": 0.8214996929147047,
|
|
"grad_norm": 7.52798430069717,
|
|
"learning_rate": 4.6973368523003376e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22070
|
|
},
|
|
{
|
|
"epoch": 0.8218719175150285,
|
|
"grad_norm": 7.125079901693356,
|
|
"learning_rate": 4.6783992832529845e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3771,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22080
|
|
},
|
|
{
|
|
"epoch": 0.8222441421153523,
|
|
"grad_norm": 7.468277760582939,
|
|
"learning_rate": 4.6594960239001975e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3987,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22090
|
|
},
|
|
{
|
|
"epoch": 0.8226163667156763,
|
|
"grad_norm": 8.682157675578885,
|
|
"learning_rate": 4.6406271061570166e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3907,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22100
|
|
},
|
|
{
|
|
"epoch": 0.8229885913160001,
|
|
"grad_norm": 8.22289791801796,
|
|
"learning_rate": 4.6217925618805356e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4072,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 22110
|
|
},
|
|
{
|
|
"epoch": 0.8233608159163239,
|
|
"grad_norm": 8.417515595967798,
|
|
"learning_rate": 4.6029924228698e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22120
|
|
},
|
|
{
|
|
"epoch": 0.8237330405166478,
|
|
"grad_norm": 8.174590374786446,
|
|
"learning_rate": 4.5842267208657524e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3881,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.4921875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22130
|
|
},
|
|
{
|
|
"epoch": 0.8241052651169716,
|
|
"grad_norm": 7.717316827041395,
|
|
"learning_rate": 4.565495487551213e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 22140
|
|
},
|
|
{
|
|
"epoch": 0.8244774897172954,
|
|
"grad_norm": 7.153228359071103,
|
|
"learning_rate": 4.546798754550801e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3783,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22150
|
|
},
|
|
{
|
|
"epoch": 0.8248497143176192,
|
|
"grad_norm": 6.301500768014381,
|
|
"learning_rate": 4.5281365534308946e-08,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.361,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22160
|
|
},
|
|
{
|
|
"epoch": 0.825221938917943,
|
|
"grad_norm": 8.123556255703015,
|
|
"learning_rate": 4.5095089156995505e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 22170
|
|
},
|
|
{
|
|
"epoch": 0.825594163518267,
|
|
"grad_norm": 7.9570480226746065,
|
|
"learning_rate": 4.490915872806497e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22180
|
|
},
|
|
{
|
|
"epoch": 0.8259663881185908,
|
|
"grad_norm": 6.846461442319267,
|
|
"learning_rate": 4.4723574561430246e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.377,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22190
|
|
},
|
|
{
|
|
"epoch": 0.8263386127189146,
|
|
"grad_norm": 7.868321533174571,
|
|
"learning_rate": 4.4538336970420004e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.367,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22200
|
|
},
|
|
{
|
|
"epoch": 0.8267108373192384,
|
|
"grad_norm": 8.347869507705378,
|
|
"learning_rate": 4.435344626777754e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22210
|
|
},
|
|
{
|
|
"epoch": 0.8270830619195623,
|
|
"grad_norm": 7.543485546709582,
|
|
"learning_rate": 4.41689027656604e-08,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3899,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22220
|
|
},
|
|
{
|
|
"epoch": 0.8274552865198861,
|
|
"grad_norm": 6.228983064570383,
|
|
"learning_rate": 4.398470677564023e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3509,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 22230
|
|
},
|
|
{
|
|
"epoch": 0.8278275111202099,
|
|
"grad_norm": 8.35609612663197,
|
|
"learning_rate": 4.380085860870156e-08,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3797,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.28125,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22240
|
|
},
|
|
{
|
|
"epoch": 0.8281997357205337,
|
|
"grad_norm": 7.966941657596074,
|
|
"learning_rate": 4.361735857524221e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22250
|
|
},
|
|
{
|
|
"epoch": 0.8285719603208576,
|
|
"grad_norm": 7.2640460047311475,
|
|
"learning_rate": 4.3434206985071667e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3669,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22260
|
|
},
|
|
{
|
|
"epoch": 0.8289441849211815,
|
|
"grad_norm": 8.261891897222814,
|
|
"learning_rate": 4.325140414741152e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22270
|
|
},
|
|
{
|
|
"epoch": 0.8293164095215053,
|
|
"grad_norm": 9.044246753915317,
|
|
"learning_rate": 4.3068950370894476e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3924,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22280
|
|
},
|
|
{
|
|
"epoch": 0.8296886341218291,
|
|
"grad_norm": 8.491624801027719,
|
|
"learning_rate": 4.288684596356365e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22290
|
|
},
|
|
{
|
|
"epoch": 0.830060858722153,
|
|
"grad_norm": 6.959684184500833,
|
|
"learning_rate": 4.270509123287277e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3757,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22300
|
|
},
|
|
{
|
|
"epoch": 0.8304330833224768,
|
|
"grad_norm": 9.07389794124682,
|
|
"learning_rate": 4.2523686485684745e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3933,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22310
|
|
},
|
|
{
|
|
"epoch": 0.8308053079228006,
|
|
"grad_norm": 7.563688895324976,
|
|
"learning_rate": 4.23426320282719e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22320
|
|
},
|
|
{
|
|
"epoch": 0.8311775325231244,
|
|
"grad_norm": 8.656674883434132,
|
|
"learning_rate": 4.2161928166314834e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3982,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22330
|
|
},
|
|
{
|
|
"epoch": 0.8315497571234483,
|
|
"grad_norm": 8.302885191081353,
|
|
"learning_rate": 4.1981575204902695e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3803,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22340
|
|
},
|
|
{
|
|
"epoch": 0.8319219817237721,
|
|
"grad_norm": 9.02630641282946,
|
|
"learning_rate": 4.1801573448531744e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3748,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22350
|
|
},
|
|
{
|
|
"epoch": 0.832294206324096,
|
|
"grad_norm": 7.9657130538197425,
|
|
"learning_rate": 4.1621923201105466e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3768,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22360
|
|
},
|
|
{
|
|
"epoch": 0.8326664309244198,
|
|
"grad_norm": 7.5218784421471305,
|
|
"learning_rate": 4.144262476593402e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4059,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22370
|
|
},
|
|
{
|
|
"epoch": 0.8330386555247437,
|
|
"grad_norm": 7.607201302295192,
|
|
"learning_rate": 4.126367844573322e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22380
|
|
},
|
|
{
|
|
"epoch": 0.8334108801250675,
|
|
"grad_norm": 7.126082588010973,
|
|
"learning_rate": 4.108508454262485e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -908.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3758,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22390
|
|
},
|
|
{
|
|
"epoch": 0.8337831047253913,
|
|
"grad_norm": 7.683370956993338,
|
|
"learning_rate": 4.090684335813532e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22400
|
|
},
|
|
{
|
|
"epoch": 0.8341553293257151,
|
|
"grad_norm": 9.137378474844006,
|
|
"learning_rate": 4.0728955193195803e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3938,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22410
|
|
},
|
|
{
|
|
"epoch": 0.834527553926039,
|
|
"grad_norm": 8.118773865198266,
|
|
"learning_rate": 4.055142034814119e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22420
|
|
},
|
|
{
|
|
"epoch": 0.8348997785263628,
|
|
"grad_norm": 7.841843716954336,
|
|
"learning_rate": 4.0374239122710015e-08,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3748,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22430
|
|
},
|
|
{
|
|
"epoch": 0.8352720031266866,
|
|
"grad_norm": 7.1117943514581246,
|
|
"learning_rate": 4.019741181604383e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3899,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 22440
|
|
},
|
|
{
|
|
"epoch": 0.8356442277270105,
|
|
"grad_norm": 8.531937827357865,
|
|
"learning_rate": 4.002093872668655e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4012,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22450
|
|
},
|
|
{
|
|
"epoch": 0.8360164523273343,
|
|
"grad_norm": 9.675083514282063,
|
|
"learning_rate": 3.984482015258411e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22460
|
|
},
|
|
{
|
|
"epoch": 0.8363886769276582,
|
|
"grad_norm": 7.597356961381909,
|
|
"learning_rate": 3.966905639108384e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22470
|
|
},
|
|
{
|
|
"epoch": 0.836760901527982,
|
|
"grad_norm": 10.383403491564623,
|
|
"learning_rate": 3.9493647738934046e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22480
|
|
},
|
|
{
|
|
"epoch": 0.8371331261283058,
|
|
"grad_norm": 7.882158133419659,
|
|
"learning_rate": 3.931859449228353e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3804,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22490
|
|
},
|
|
{
|
|
"epoch": 0.8375053507286296,
|
|
"grad_norm": 7.124671995046582,
|
|
"learning_rate": 3.91438969466811e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3813,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22500
|
|
},
|
|
{
|
|
"epoch": 0.8378775753289535,
|
|
"grad_norm": 6.495335600889642,
|
|
"learning_rate": 3.896955539707483e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3764,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22510
|
|
},
|
|
{
|
|
"epoch": 0.8382497999292773,
|
|
"grad_norm": 7.316658568467803,
|
|
"learning_rate": 3.879557013781193e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3779,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22520
|
|
},
|
|
{
|
|
"epoch": 0.8386220245296011,
|
|
"grad_norm": 8.580164796623903,
|
|
"learning_rate": 3.862194146263803e-08,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3704,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22530
|
|
},
|
|
{
|
|
"epoch": 0.838994249129925,
|
|
"grad_norm": 7.586724906713899,
|
|
"learning_rate": 3.844866966469668e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22540
|
|
},
|
|
{
|
|
"epoch": 0.8393664737302489,
|
|
"grad_norm": 7.845240726169775,
|
|
"learning_rate": 3.827575503652902e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22550
|
|
},
|
|
{
|
|
"epoch": 0.8397386983305727,
|
|
"grad_norm": 7.7590305447598205,
|
|
"learning_rate": 3.810319787007293e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3769,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 22560
|
|
},
|
|
{
|
|
"epoch": 0.8401109229308965,
|
|
"grad_norm": 7.622035136455193,
|
|
"learning_rate": 3.7930998456663033e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.366,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22570
|
|
},
|
|
{
|
|
"epoch": 0.8404831475312203,
|
|
"grad_norm": 7.708780205138886,
|
|
"learning_rate": 3.7759157087029784e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3864,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22580
|
|
},
|
|
{
|
|
"epoch": 0.8408553721315442,
|
|
"grad_norm": 9.797825836893972,
|
|
"learning_rate": 3.7587674051299254e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4077,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22590
|
|
},
|
|
{
|
|
"epoch": 0.841227596731868,
|
|
"grad_norm": 8.320328744592516,
|
|
"learning_rate": 3.7416549638992406e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3994,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 22600
|
|
},
|
|
{
|
|
"epoch": 0.8415998213321918,
|
|
"grad_norm": 7.871700911717256,
|
|
"learning_rate": 3.724578413902477e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4064,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22610
|
|
},
|
|
{
|
|
"epoch": 0.8419720459325157,
|
|
"grad_norm": 8.156098474488745,
|
|
"learning_rate": 3.7075377839705964e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3716,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 22620
|
|
},
|
|
{
|
|
"epoch": 0.8423442705328396,
|
|
"grad_norm": 8.117598904106481,
|
|
"learning_rate": 3.690533102873911e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3801,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22630
|
|
},
|
|
{
|
|
"epoch": 0.8427164951331634,
|
|
"grad_norm": 7.117670798521576,
|
|
"learning_rate": 3.673564399322046e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22640
|
|
},
|
|
{
|
|
"epoch": 0.8430887197334872,
|
|
"grad_norm": 8.160824509075569,
|
|
"learning_rate": 3.6566317019638634e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22650
|
|
},
|
|
{
|
|
"epoch": 0.843460944333811,
|
|
"grad_norm": 6.484828230146329,
|
|
"learning_rate": 3.639735039387462e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3739,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22660
|
|
},
|
|
{
|
|
"epoch": 0.8438331689341348,
|
|
"grad_norm": 7.343522430921776,
|
|
"learning_rate": 3.6228744401200826e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3942,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22670
|
|
},
|
|
{
|
|
"epoch": 0.8442053935344587,
|
|
"grad_norm": 8.362784427150626,
|
|
"learning_rate": 3.606049932628094e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22680
|
|
},
|
|
{
|
|
"epoch": 0.8445776181347825,
|
|
"grad_norm": 7.295233228073698,
|
|
"learning_rate": 3.58926154531691e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3709,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22690
|
|
},
|
|
{
|
|
"epoch": 0.8449498427351063,
|
|
"grad_norm": 8.899360892015945,
|
|
"learning_rate": 3.572509306530977e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22700
|
|
},
|
|
{
|
|
"epoch": 0.8453220673354302,
|
|
"grad_norm": 9.482187948255358,
|
|
"learning_rate": 3.555793244553712e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3824,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22710
|
|
},
|
|
{
|
|
"epoch": 0.8456942919357541,
|
|
"grad_norm": 7.379907883322116,
|
|
"learning_rate": 3.53911338760744e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22720
|
|
},
|
|
{
|
|
"epoch": 0.8460665165360779,
|
|
"grad_norm": 7.2690408486986104,
|
|
"learning_rate": 3.522469763853381e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4049,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22730
|
|
},
|
|
{
|
|
"epoch": 0.8464387411364017,
|
|
"grad_norm": 8.32178210243924,
|
|
"learning_rate": 3.5058624013915516e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3719,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22740
|
|
},
|
|
{
|
|
"epoch": 0.8468109657367255,
|
|
"grad_norm": 7.398193832530898,
|
|
"learning_rate": 3.4892913282607714e-08,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -996.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3984,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22750
|
|
},
|
|
{
|
|
"epoch": 0.8471831903370494,
|
|
"grad_norm": 9.962494581890741,
|
|
"learning_rate": 3.472756572438579e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22760
|
|
},
|
|
{
|
|
"epoch": 0.8475554149373732,
|
|
"grad_norm": 7.348229020566628,
|
|
"learning_rate": 3.4562581618412136e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3903,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 22770
|
|
},
|
|
{
|
|
"epoch": 0.847927639537697,
|
|
"grad_norm": 7.627128800748097,
|
|
"learning_rate": 3.4397961243235244e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3787,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22780
|
|
},
|
|
{
|
|
"epoch": 0.8482998641380208,
|
|
"grad_norm": 8.711260914125614,
|
|
"learning_rate": 3.423370487678972e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3874,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22790
|
|
},
|
|
{
|
|
"epoch": 0.8486720887383448,
|
|
"grad_norm": 7.273308791548469,
|
|
"learning_rate": 3.406981279639554e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4157,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22800
|
|
},
|
|
{
|
|
"epoch": 0.8490443133386686,
|
|
"grad_norm": 7.957680683259726,
|
|
"learning_rate": 3.390628527875764e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3714,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22810
|
|
},
|
|
{
|
|
"epoch": 0.8494165379389924,
|
|
"grad_norm": 7.7130527594845395,
|
|
"learning_rate": 3.374312259996548e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3777,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 22820
|
|
},
|
|
{
|
|
"epoch": 0.8497887625393162,
|
|
"grad_norm": 7.61123788943586,
|
|
"learning_rate": 3.358032503549246e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3827,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22830
|
|
},
|
|
{
|
|
"epoch": 0.8501609871396401,
|
|
"grad_norm": 6.549105337711992,
|
|
"learning_rate": 3.3417892860195615e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3575,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22840
|
|
},
|
|
{
|
|
"epoch": 0.8505332117399639,
|
|
"grad_norm": 8.47965138644324,
|
|
"learning_rate": 3.325582634831508e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3743,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 22850
|
|
},
|
|
{
|
|
"epoch": 0.8509054363402877,
|
|
"grad_norm": 7.332965523233882,
|
|
"learning_rate": 3.309412577347365e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.351,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22860
|
|
},
|
|
{
|
|
"epoch": 0.8512776609406115,
|
|
"grad_norm": 9.462319450709447,
|
|
"learning_rate": 3.2932791408676275e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3718,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22870
|
|
},
|
|
{
|
|
"epoch": 0.8516498855409353,
|
|
"grad_norm": 8.82471362907676,
|
|
"learning_rate": 3.277182352630953e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22880
|
|
},
|
|
{
|
|
"epoch": 0.8520221101412593,
|
|
"grad_norm": 7.019266298571786,
|
|
"learning_rate": 3.261122239814135e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3547,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 22890
|
|
},
|
|
{
|
|
"epoch": 0.8523943347415831,
|
|
"grad_norm": 7.158767816851575,
|
|
"learning_rate": 3.2450988295320446e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3673,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22900
|
|
},
|
|
{
|
|
"epoch": 0.8527665593419069,
|
|
"grad_norm": 6.9986578880916,
|
|
"learning_rate": 3.229112148837593e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22910
|
|
},
|
|
{
|
|
"epoch": 0.8531387839422307,
|
|
"grad_norm": 8.245090670574642,
|
|
"learning_rate": 3.2131622247216576e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3699,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 22920
|
|
},
|
|
{
|
|
"epoch": 0.8535110085425546,
|
|
"grad_norm": 7.673986481554118,
|
|
"learning_rate": 3.197249084113082e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3798,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 22930
|
|
},
|
|
{
|
|
"epoch": 0.8538832331428784,
|
|
"grad_norm": 7.623529915182672,
|
|
"learning_rate": 3.1813727538785943e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3718,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22940
|
|
},
|
|
{
|
|
"epoch": 0.8542554577432022,
|
|
"grad_norm": 8.921253964679835,
|
|
"learning_rate": 3.165533260822784e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3778,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22950
|
|
},
|
|
{
|
|
"epoch": 0.854627682343526,
|
|
"grad_norm": 7.103534054248236,
|
|
"learning_rate": 3.1497306316880385e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3683,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 22960
|
|
},
|
|
{
|
|
"epoch": 0.85499990694385,
|
|
"grad_norm": 6.969367244057573,
|
|
"learning_rate": 3.133964893154503e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3731,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22970
|
|
},
|
|
{
|
|
"epoch": 0.8553721315441738,
|
|
"grad_norm": 8.06892979281593,
|
|
"learning_rate": 3.118236071840052e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.395,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 22980
|
|
},
|
|
{
|
|
"epoch": 0.8557443561444976,
|
|
"grad_norm": 9.208066767830395,
|
|
"learning_rate": 3.102544194300219e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3648,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 22990
|
|
},
|
|
{
|
|
"epoch": 0.8561165807448214,
|
|
"grad_norm": 7.212677262679424,
|
|
"learning_rate": 3.0868892870281774e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 23000
|
|
},
|
|
{
|
|
"epoch": 0.8564888053451453,
|
|
"grad_norm": 9.101058956163568,
|
|
"learning_rate": 3.071271376454665e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3713,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 23010
|
|
},
|
|
{
|
|
"epoch": 0.8568610299454691,
|
|
"grad_norm": 8.03563951569275,
|
|
"learning_rate": 3.0556904889479736e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23020
|
|
},
|
|
{
|
|
"epoch": 0.8572332545457929,
|
|
"grad_norm": 6.814225172404424,
|
|
"learning_rate": 3.040146650813866e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23030
|
|
},
|
|
{
|
|
"epoch": 0.8576054791461167,
|
|
"grad_norm": 8.333212005948749,
|
|
"learning_rate": 3.02463988829558e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4147,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23040
|
|
},
|
|
{
|
|
"epoch": 0.8579777037464406,
|
|
"grad_norm": 6.577318915619248,
|
|
"learning_rate": 3.009170227573749e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3603,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23050
|
|
},
|
|
{
|
|
"epoch": 0.8583499283467645,
|
|
"grad_norm": 8.345519831736928,
|
|
"learning_rate": 2.993737694766349e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.378,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23060
|
|
},
|
|
{
|
|
"epoch": 0.8587221529470883,
|
|
"grad_norm": 7.106137777326428,
|
|
"learning_rate": 2.978342315928692e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3749,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23070
|
|
},
|
|
{
|
|
"epoch": 0.8590943775474121,
|
|
"grad_norm": 7.831237999774117,
|
|
"learning_rate": 2.9629841170533408e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23080
|
|
},
|
|
{
|
|
"epoch": 0.859466602147736,
|
|
"grad_norm": 8.207317216391415,
|
|
"learning_rate": 2.9476631240701212e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3741,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23090
|
|
},
|
|
{
|
|
"epoch": 0.8598388267480598,
|
|
"grad_norm": 7.810936204306551,
|
|
"learning_rate": 2.9323793628460018e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3742,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23100
|
|
},
|
|
{
|
|
"epoch": 0.8602110513483836,
|
|
"grad_norm": 7.283605583703448,
|
|
"learning_rate": 2.917132859185123e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.4008,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.3125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 23110
|
|
},
|
|
{
|
|
"epoch": 0.8605832759487074,
|
|
"grad_norm": 8.448888205617436,
|
|
"learning_rate": 2.9019236388286907e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3801,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23120
|
|
},
|
|
{
|
|
"epoch": 0.8609555005490312,
|
|
"grad_norm": 8.956720361124416,
|
|
"learning_rate": 2.886751727455003e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4093,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23130
|
|
},
|
|
{
|
|
"epoch": 0.8613277251493551,
|
|
"grad_norm": 6.9905873776980565,
|
|
"learning_rate": 2.8716171506793423e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23140
|
|
},
|
|
{
|
|
"epoch": 0.861699949749679,
|
|
"grad_norm": 8.09554158849423,
|
|
"learning_rate": 2.8565199340539553e-08,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23150
|
|
},
|
|
{
|
|
"epoch": 0.8620721743500028,
|
|
"grad_norm": 7.31196522247097,
|
|
"learning_rate": 2.8414601030680273e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23160
|
|
},
|
|
{
|
|
"epoch": 0.8624443989503267,
|
|
"grad_norm": 8.408857068081785,
|
|
"learning_rate": 2.8264376831476034e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23170
|
|
},
|
|
{
|
|
"epoch": 0.8628166235506505,
|
|
"grad_norm": 7.205928335248589,
|
|
"learning_rate": 2.8114526996556002e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23180
|
|
},
|
|
{
|
|
"epoch": 0.8631888481509743,
|
|
"grad_norm": 8.240678327152986,
|
|
"learning_rate": 2.796505177891692e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23190
|
|
},
|
|
{
|
|
"epoch": 0.8635610727512981,
|
|
"grad_norm": 7.38877636204887,
|
|
"learning_rate": 2.7815951430923247e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3826,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 23200
|
|
},
|
|
{
|
|
"epoch": 0.8639332973516219,
|
|
"grad_norm": 8.914270361951237,
|
|
"learning_rate": 2.7667226204306588e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3933,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23210
|
|
},
|
|
{
|
|
"epoch": 0.8643055219519458,
|
|
"grad_norm": 7.861453995045666,
|
|
"learning_rate": 2.7518876350164923e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3838,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23220
|
|
},
|
|
{
|
|
"epoch": 0.8646777465522696,
|
|
"grad_norm": 8.580219418760938,
|
|
"learning_rate": 2.737090211896295e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3859,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23230
|
|
},
|
|
{
|
|
"epoch": 0.8650499711525935,
|
|
"grad_norm": 7.433476369683042,
|
|
"learning_rate": 2.7223303760530726e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23240
|
|
},
|
|
{
|
|
"epoch": 0.8654221957529173,
|
|
"grad_norm": 8.076347812914173,
|
|
"learning_rate": 2.7076081524064016e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23250
|
|
},
|
|
{
|
|
"epoch": 0.8657944203532412,
|
|
"grad_norm": 8.26603733190942,
|
|
"learning_rate": 2.692923565812333e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3788,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23260
|
|
},
|
|
{
|
|
"epoch": 0.866166644953565,
|
|
"grad_norm": 7.3932861877936,
|
|
"learning_rate": 2.678276641063404e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3681,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23270
|
|
},
|
|
{
|
|
"epoch": 0.8665388695538888,
|
|
"grad_norm": 7.90787070798269,
|
|
"learning_rate": 2.663667402888531e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3894,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23280
|
|
},
|
|
{
|
|
"epoch": 0.8669110941542126,
|
|
"grad_norm": 7.653683107981299,
|
|
"learning_rate": 2.649095875953028e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23290
|
|
},
|
|
{
|
|
"epoch": 0.8672833187545365,
|
|
"grad_norm": 7.728534376685968,
|
|
"learning_rate": 2.634562084858538e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3887,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23300
|
|
},
|
|
{
|
|
"epoch": 0.8676555433548603,
|
|
"grad_norm": 8.18399958344198,
|
|
"learning_rate": 2.6200660541429692e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 23310
|
|
},
|
|
{
|
|
"epoch": 0.8680277679551841,
|
|
"grad_norm": 9.211955918051675,
|
|
"learning_rate": 2.605607808280516e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3927,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23320
|
|
},
|
|
{
|
|
"epoch": 0.868399992555508,
|
|
"grad_norm": 7.473741876285765,
|
|
"learning_rate": 2.5911873716815442e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3812,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23330
|
|
},
|
|
{
|
|
"epoch": 0.8687722171558319,
|
|
"grad_norm": 7.290766765274376,
|
|
"learning_rate": 2.5768047686926086e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.373,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23340
|
|
},
|
|
{
|
|
"epoch": 0.8691444417561557,
|
|
"grad_norm": 7.014004035055537,
|
|
"learning_rate": 2.562460023596369e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -904.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23350
|
|
},
|
|
{
|
|
"epoch": 0.8695166663564795,
|
|
"grad_norm": 6.138912187512264,
|
|
"learning_rate": 2.5481531606115825e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3953,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23360
|
|
},
|
|
{
|
|
"epoch": 0.8698888909568033,
|
|
"grad_norm": 7.700216938163198,
|
|
"learning_rate": 2.533884203893044e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3847,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23370
|
|
},
|
|
{
|
|
"epoch": 0.8702611155571272,
|
|
"grad_norm": 7.932943381182641,
|
|
"learning_rate": 2.5196531775315528e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23380
|
|
},
|
|
{
|
|
"epoch": 0.870633340157451,
|
|
"grad_norm": 8.37961661320531,
|
|
"learning_rate": 2.5054601055538667e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3653,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23390
|
|
},
|
|
{
|
|
"epoch": 0.8710055647577748,
|
|
"grad_norm": 7.094534728876871,
|
|
"learning_rate": 2.4913050119226563e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23400
|
|
},
|
|
{
|
|
"epoch": 0.8713777893580987,
|
|
"grad_norm": 7.3737592462450445,
|
|
"learning_rate": 2.4771879205364832e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.96875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23410
|
|
},
|
|
{
|
|
"epoch": 0.8717500139584226,
|
|
"grad_norm": 7.205929564882144,
|
|
"learning_rate": 2.4631088552297423e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23420
|
|
},
|
|
{
|
|
"epoch": 0.8721222385587464,
|
|
"grad_norm": 6.873152793427172,
|
|
"learning_rate": 2.4490678397726396e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3884,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 23430
|
|
},
|
|
{
|
|
"epoch": 0.8724944631590702,
|
|
"grad_norm": 8.306318041678262,
|
|
"learning_rate": 2.4350648978711142e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3734,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23440
|
|
},
|
|
{
|
|
"epoch": 0.872866687759394,
|
|
"grad_norm": 7.23622862453243,
|
|
"learning_rate": 2.421100053166844e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3855,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23450
|
|
},
|
|
{
|
|
"epoch": 0.8732389123597178,
|
|
"grad_norm": 6.911733646305349,
|
|
"learning_rate": 2.4071733292371944e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3657,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23460
|
|
},
|
|
{
|
|
"epoch": 0.8736111369600417,
|
|
"grad_norm": 7.7273208877952335,
|
|
"learning_rate": 2.3932847495951453e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3677,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23470
|
|
},
|
|
{
|
|
"epoch": 0.8739833615603655,
|
|
"grad_norm": 7.805520841378881,
|
|
"learning_rate": 2.3794343376892982e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3725,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23480
|
|
},
|
|
{
|
|
"epoch": 0.8743555861606893,
|
|
"grad_norm": 7.1890656429529765,
|
|
"learning_rate": 2.3656221169037926e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3908,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23490
|
|
},
|
|
{
|
|
"epoch": 0.8747278107610132,
|
|
"grad_norm": 7.594624924416488,
|
|
"learning_rate": 2.3518481105583104e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23500
|
|
},
|
|
{
|
|
"epoch": 0.8751000353613371,
|
|
"grad_norm": 7.592768940512571,
|
|
"learning_rate": 2.3381123419080022e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3825,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23510
|
|
},
|
|
{
|
|
"epoch": 0.8754722599616609,
|
|
"grad_norm": 7.54311539584731,
|
|
"learning_rate": 2.3244148341434707e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3975,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23520
|
|
},
|
|
{
|
|
"epoch": 0.8758444845619847,
|
|
"grad_norm": 8.062243061347452,
|
|
"learning_rate": 2.3107556103907022e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3745,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23530
|
|
},
|
|
{
|
|
"epoch": 0.8762167091623085,
|
|
"grad_norm": 8.654284974227366,
|
|
"learning_rate": 2.2971346937110668e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23540
|
|
},
|
|
{
|
|
"epoch": 0.8765889337626324,
|
|
"grad_norm": 7.408193102471745,
|
|
"learning_rate": 2.283552107101247e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3903,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23550
|
|
},
|
|
{
|
|
"epoch": 0.8769611583629562,
|
|
"grad_norm": 8.16182097563237,
|
|
"learning_rate": 2.270007873493221e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4074,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 23560
|
|
},
|
|
{
|
|
"epoch": 0.87733338296328,
|
|
"grad_norm": 8.141436264300257,
|
|
"learning_rate": 2.256502015754211e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4098,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23570
|
|
},
|
|
{
|
|
"epoch": 0.8777056075636038,
|
|
"grad_norm": 6.2790713241749625,
|
|
"learning_rate": 2.2430345566866378e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3858,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23580
|
|
},
|
|
{
|
|
"epoch": 0.8780778321639278,
|
|
"grad_norm": 7.194782286168086,
|
|
"learning_rate": 2.2296055190281023e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23590
|
|
},
|
|
{
|
|
"epoch": 0.8784500567642516,
|
|
"grad_norm": 7.4288289830284056,
|
|
"learning_rate": 2.2162149254513374e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23600
|
|
},
|
|
{
|
|
"epoch": 0.8788222813645754,
|
|
"grad_norm": 6.843606252034596,
|
|
"learning_rate": 2.2028627985641702e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3759,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23610
|
|
},
|
|
{
|
|
"epoch": 0.8791945059648992,
|
|
"grad_norm": 9.38889983863122,
|
|
"learning_rate": 2.1895491609094735e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3787,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23620
|
|
},
|
|
{
|
|
"epoch": 0.879566730565223,
|
|
"grad_norm": 7.388067456628311,
|
|
"learning_rate": 2.1762740349651476e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.394,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 23630
|
|
},
|
|
{
|
|
"epoch": 0.8799389551655469,
|
|
"grad_norm": 8.334577164238185,
|
|
"learning_rate": 2.1630374431440668e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3668,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23640
|
|
},
|
|
{
|
|
"epoch": 0.8803111797658707,
|
|
"grad_norm": 8.25334846417751,
|
|
"learning_rate": 2.149839407794052e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3706,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23650
|
|
},
|
|
{
|
|
"epoch": 0.8806834043661945,
|
|
"grad_norm": 8.858771356979142,
|
|
"learning_rate": 2.136679951197823e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.357,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23660
|
|
},
|
|
{
|
|
"epoch": 0.8810556289665183,
|
|
"grad_norm": 9.157186885086906,
|
|
"learning_rate": 2.1235590955729632e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 23670
|
|
},
|
|
{
|
|
"epoch": 0.8814278535668423,
|
|
"grad_norm": 8.37489451836421,
|
|
"learning_rate": 2.1104768630718915e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3918,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23680
|
|
},
|
|
{
|
|
"epoch": 0.8818000781671661,
|
|
"grad_norm": 7.8975686181123494,
|
|
"learning_rate": 2.0974332757818153e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3735,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 23690
|
|
},
|
|
{
|
|
"epoch": 0.8821723027674899,
|
|
"grad_norm": 8.53542140826091,
|
|
"learning_rate": 2.084428355724699e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3846,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23700
|
|
},
|
|
{
|
|
"epoch": 0.8825445273678137,
|
|
"grad_norm": 7.268764030850396,
|
|
"learning_rate": 2.0714621248572124e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23710
|
|
},
|
|
{
|
|
"epoch": 0.8829167519681376,
|
|
"grad_norm": 6.7019545583869995,
|
|
"learning_rate": 2.0585346050707138e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3723,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23720
|
|
},
|
|
{
|
|
"epoch": 0.8832889765684614,
|
|
"grad_norm": 8.868884881060872,
|
|
"learning_rate": 2.045645818191208e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3726,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23730
|
|
},
|
|
{
|
|
"epoch": 0.8836612011687852,
|
|
"grad_norm": 8.8069038457951,
|
|
"learning_rate": 2.0327957859792967e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3915,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23740
|
|
},
|
|
{
|
|
"epoch": 0.884033425769109,
|
|
"grad_norm": 7.922256801637617,
|
|
"learning_rate": 2.0199845301301616e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23750
|
|
},
|
|
{
|
|
"epoch": 0.8844056503694329,
|
|
"grad_norm": 8.836060818077438,
|
|
"learning_rate": 2.007212072273501e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3883,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23760
|
|
},
|
|
{
|
|
"epoch": 0.8847778749697568,
|
|
"grad_norm": 7.68012954959226,
|
|
"learning_rate": 1.9944784339735232e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3946,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23770
|
|
},
|
|
{
|
|
"epoch": 0.8851500995700806,
|
|
"grad_norm": 7.386902077130625,
|
|
"learning_rate": 1.981783636728887e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3648,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23780
|
|
},
|
|
{
|
|
"epoch": 0.8855223241704044,
|
|
"grad_norm": 7.844205529173265,
|
|
"learning_rate": 1.9691277019726838e-08,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3758,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23790
|
|
},
|
|
{
|
|
"epoch": 0.8858945487707283,
|
|
"grad_norm": 7.3896844120209755,
|
|
"learning_rate": 1.956510651072385e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3821,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23800
|
|
},
|
|
{
|
|
"epoch": 0.8862667733710521,
|
|
"grad_norm": 8.839334941412588,
|
|
"learning_rate": 1.9439325053298123e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3948,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23810
|
|
},
|
|
{
|
|
"epoch": 0.8866389979713759,
|
|
"grad_norm": 8.149869479978236,
|
|
"learning_rate": 1.9313932859811033e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3981,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 23820
|
|
},
|
|
{
|
|
"epoch": 0.8870112225716997,
|
|
"grad_norm": 7.426579291784634,
|
|
"learning_rate": 1.918893014196682e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3744,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23830
|
|
},
|
|
{
|
|
"epoch": 0.8873834471720236,
|
|
"grad_norm": 7.942050498929881,
|
|
"learning_rate": 1.9064317110812106e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3794,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 23840
|
|
},
|
|
{
|
|
"epoch": 0.8877556717723475,
|
|
"grad_norm": 6.8411931373414,
|
|
"learning_rate": 1.894009397673549e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3888,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23850
|
|
},
|
|
{
|
|
"epoch": 0.8881278963726713,
|
|
"grad_norm": 8.321957083956208,
|
|
"learning_rate": 1.881626094946745e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23860
|
|
},
|
|
{
|
|
"epoch": 0.8885001209729951,
|
|
"grad_norm": 7.557394710264497,
|
|
"learning_rate": 1.8692818238079778e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23870
|
|
},
|
|
{
|
|
"epoch": 0.888872345573319,
|
|
"grad_norm": 8.745715395649372,
|
|
"learning_rate": 1.8569766050985257e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23880
|
|
},
|
|
{
|
|
"epoch": 0.8892445701736428,
|
|
"grad_norm": 8.548067490512263,
|
|
"learning_rate": 1.8447104595937428e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3688,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 23890
|
|
},
|
|
{
|
|
"epoch": 0.8896167947739666,
|
|
"grad_norm": 7.064093113221702,
|
|
"learning_rate": 1.832483408002994e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3972,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 23900
|
|
},
|
|
{
|
|
"epoch": 0.8899890193742904,
|
|
"grad_norm": 5.937762730546113,
|
|
"learning_rate": 1.82029547096966e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3741,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 23910
|
|
},
|
|
{
|
|
"epoch": 0.8903612439746142,
|
|
"grad_norm": 9.188709556339932,
|
|
"learning_rate": 1.8081466690710755e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3924,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23920
|
|
},
|
|
{
|
|
"epoch": 0.8907334685749381,
|
|
"grad_norm": 8.208859637999655,
|
|
"learning_rate": 1.7960370228185083e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3879,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 23930
|
|
},
|
|
{
|
|
"epoch": 0.891105693175262,
|
|
"grad_norm": 8.0867336952243,
|
|
"learning_rate": 1.7839665526571014e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3655,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 23940
|
|
},
|
|
{
|
|
"epoch": 0.8914779177755858,
|
|
"grad_norm": 7.945179456838408,
|
|
"learning_rate": 1.7719352789658775e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3912,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 23950
|
|
},
|
|
{
|
|
"epoch": 0.8918501423759096,
|
|
"grad_norm": 7.773904770673485,
|
|
"learning_rate": 1.759943222057661e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3641,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 23960
|
|
},
|
|
{
|
|
"epoch": 0.8922223669762335,
|
|
"grad_norm": 12.334238987153517,
|
|
"learning_rate": 1.7479904021790863e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 23970
|
|
},
|
|
{
|
|
"epoch": 0.8925945915765573,
|
|
"grad_norm": 10.800369074258034,
|
|
"learning_rate": 1.736076839510531e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4051,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 23980
|
|
},
|
|
{
|
|
"epoch": 0.8929668161768811,
|
|
"grad_norm": 6.848124580898033,
|
|
"learning_rate": 1.7242025541660875e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4056,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 23990
|
|
},
|
|
{
|
|
"epoch": 0.8933390407772049,
|
|
"grad_norm": 6.023388663236392,
|
|
"learning_rate": 1.7123675661935483e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4011,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24000
|
|
},
|
|
{
|
|
"epoch": 0.8937112653775288,
|
|
"grad_norm": 9.035937072453944,
|
|
"learning_rate": 1.7005718955743432e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.4031,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 24010
|
|
},
|
|
{
|
|
"epoch": 0.8940834899778526,
|
|
"grad_norm": 8.506146697296975,
|
|
"learning_rate": 1.6888155622235474e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3968,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24020
|
|
},
|
|
{
|
|
"epoch": 0.8944557145781765,
|
|
"grad_norm": 7.6780244804025966,
|
|
"learning_rate": 1.6770985859897863e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3617,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24030
|
|
},
|
|
{
|
|
"epoch": 0.8948279391785003,
|
|
"grad_norm": 7.53485765991774,
|
|
"learning_rate": 1.6654209866552638e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3745,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 24040
|
|
},
|
|
{
|
|
"epoch": 0.8952001637788242,
|
|
"grad_norm": 10.153440752509923,
|
|
"learning_rate": 1.6537827839356923e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4072,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24050
|
|
},
|
|
{
|
|
"epoch": 0.895572388379148,
|
|
"grad_norm": 11.233409086728724,
|
|
"learning_rate": 1.642183997480273e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3992,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 24060
|
|
},
|
|
{
|
|
"epoch": 0.8959446129794718,
|
|
"grad_norm": 7.140429141211188,
|
|
"learning_rate": 1.630624646871659e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3977,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24070
|
|
},
|
|
{
|
|
"epoch": 0.8963168375797956,
|
|
"grad_norm": 7.1034712439088405,
|
|
"learning_rate": 1.619104751625913e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3453,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24080
|
|
},
|
|
{
|
|
"epoch": 0.8966890621801195,
|
|
"grad_norm": 6.901548908548239,
|
|
"learning_rate": 1.6076243311924974e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 24090
|
|
},
|
|
{
|
|
"epoch": 0.8970612867804433,
|
|
"grad_norm": 7.990183474820668,
|
|
"learning_rate": 1.5961834049542154e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4108,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24100
|
|
},
|
|
{
|
|
"epoch": 0.8974335113807671,
|
|
"grad_norm": 8.69182961715929,
|
|
"learning_rate": 1.5847819922272077e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3668,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.375,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24110
|
|
},
|
|
{
|
|
"epoch": 0.897805735981091,
|
|
"grad_norm": 7.569312796369849,
|
|
"learning_rate": 1.573420112260884e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3778,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24120
|
|
},
|
|
{
|
|
"epoch": 0.8981779605814149,
|
|
"grad_norm": 7.981116300815327,
|
|
"learning_rate": 1.56209778423792e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3799,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 24130
|
|
},
|
|
{
|
|
"epoch": 0.8985501851817387,
|
|
"grad_norm": 7.2871933662215165,
|
|
"learning_rate": 1.550815027274216e-08,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3882,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24140
|
|
},
|
|
{
|
|
"epoch": 0.8989224097820625,
|
|
"grad_norm": 7.926164430151094,
|
|
"learning_rate": 1.5395718604188506e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.379,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 24150
|
|
},
|
|
{
|
|
"epoch": 0.8992946343823863,
|
|
"grad_norm": 7.18076729434121,
|
|
"learning_rate": 1.528368302654079e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3888,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 24160
|
|
},
|
|
{
|
|
"epoch": 0.8996668589827101,
|
|
"grad_norm": 7.842203073368581,
|
|
"learning_rate": 1.5172043728952672e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24170
|
|
},
|
|
{
|
|
"epoch": 0.900039083583034,
|
|
"grad_norm": 8.82925795491696,
|
|
"learning_rate": 1.506080089990888e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3978,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24180
|
|
},
|
|
{
|
|
"epoch": 0.9004113081833578,
|
|
"grad_norm": 7.774185692439233,
|
|
"learning_rate": 1.4949954727224627e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3943,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 24190
|
|
},
|
|
{
|
|
"epoch": 0.9007835327836817,
|
|
"grad_norm": 6.418884342528473,
|
|
"learning_rate": 1.48395053980456e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3947,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24200
|
|
},
|
|
{
|
|
"epoch": 0.9011557573840056,
|
|
"grad_norm": 6.698918956166137,
|
|
"learning_rate": 1.472945309884735e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3858,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 24210
|
|
},
|
|
{
|
|
"epoch": 0.9015279819843294,
|
|
"grad_norm": 7.063105316508377,
|
|
"learning_rate": 1.4619798015435198e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4132,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24220
|
|
},
|
|
{
|
|
"epoch": 0.9019002065846532,
|
|
"grad_norm": 7.786129072278861,
|
|
"learning_rate": 1.4510540332943799e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3524,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24230
|
|
},
|
|
{
|
|
"epoch": 0.902272431184977,
|
|
"grad_norm": 8.680869543565294,
|
|
"learning_rate": 1.4401680235836766e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3734,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24240
|
|
},
|
|
{
|
|
"epoch": 0.9026446557853008,
|
|
"grad_norm": 7.070441521993716,
|
|
"learning_rate": 1.4293217907906663e-08,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3719,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24250
|
|
},
|
|
{
|
|
"epoch": 0.9030168803856247,
|
|
"grad_norm": 8.698362364809439,
|
|
"learning_rate": 1.4185153532274313e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3787,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24260
|
|
},
|
|
{
|
|
"epoch": 0.9033891049859485,
|
|
"grad_norm": 8.475029694240135,
|
|
"learning_rate": 1.4077487291388746e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24270
|
|
},
|
|
{
|
|
"epoch": 0.9037613295862723,
|
|
"grad_norm": 7.539988979610094,
|
|
"learning_rate": 1.3970219367026692e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3742,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24280
|
|
},
|
|
{
|
|
"epoch": 0.9041335541865962,
|
|
"grad_norm": 8.321271393505489,
|
|
"learning_rate": 1.3863349940292562e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3592,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24290
|
|
},
|
|
{
|
|
"epoch": 0.9045057787869201,
|
|
"grad_norm": 7.979562253023922,
|
|
"learning_rate": 1.3756879191617831e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3739,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24300
|
|
},
|
|
{
|
|
"epoch": 0.9048780033872439,
|
|
"grad_norm": 8.017171314759302,
|
|
"learning_rate": 1.365080730076093e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3771,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24310
|
|
},
|
|
{
|
|
"epoch": 0.9052502279875677,
|
|
"grad_norm": 7.56174405381541,
|
|
"learning_rate": 1.3545134446806911e-08,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3789,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24320
|
|
},
|
|
{
|
|
"epoch": 0.9056224525878915,
|
|
"grad_norm": 8.417901950981491,
|
|
"learning_rate": 1.343986080816703e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3742,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24330
|
|
},
|
|
{
|
|
"epoch": 0.9059946771882154,
|
|
"grad_norm": 8.628040806455697,
|
|
"learning_rate": 1.3334986562578614e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -8.0625,
|
|
"rewards/margins": 1.4375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24340
|
|
},
|
|
{
|
|
"epoch": 0.9063669017885392,
|
|
"grad_norm": 8.315085163486005,
|
|
"learning_rate": 1.3230511887104639e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3652,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24350
|
|
},
|
|
{
|
|
"epoch": 0.906739126388863,
|
|
"grad_norm": 7.498081763564755,
|
|
"learning_rate": 1.3126436958133507e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3655,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24360
|
|
},
|
|
{
|
|
"epoch": 0.9071113509891868,
|
|
"grad_norm": 7.827063875341011,
|
|
"learning_rate": 1.3022761951378663e-08,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3569,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24370
|
|
},
|
|
{
|
|
"epoch": 0.9074835755895108,
|
|
"grad_norm": 8.253136513669775,
|
|
"learning_rate": 1.2919487041878369e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.392,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24380
|
|
},
|
|
{
|
|
"epoch": 0.9078558001898346,
|
|
"grad_norm": 7.346171183198129,
|
|
"learning_rate": 1.2816612403995397e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3924,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24390
|
|
},
|
|
{
|
|
"epoch": 0.9082280247901584,
|
|
"grad_norm": 7.787834835388826,
|
|
"learning_rate": 1.2714138211416758e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24400
|
|
},
|
|
{
|
|
"epoch": 0.9086002493904822,
|
|
"grad_norm": 7.3439640093776495,
|
|
"learning_rate": 1.2612064637153336e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3835,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24410
|
|
},
|
|
{
|
|
"epoch": 0.908972473990806,
|
|
"grad_norm": 7.527452404221456,
|
|
"learning_rate": 1.251039185353961e-08,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4009,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24420
|
|
},
|
|
{
|
|
"epoch": 0.9093446985911299,
|
|
"grad_norm": 8.10335194050169,
|
|
"learning_rate": 1.2409120032233433e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3831,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24430
|
|
},
|
|
{
|
|
"epoch": 0.9097169231914537,
|
|
"grad_norm": 8.733028413439426,
|
|
"learning_rate": 1.2308249344215704e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24440
|
|
},
|
|
{
|
|
"epoch": 0.9100891477917775,
|
|
"grad_norm": 7.515066260372048,
|
|
"learning_rate": 1.220777995979011e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3775,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 24450
|
|
},
|
|
{
|
|
"epoch": 0.9104613723921013,
|
|
"grad_norm": 7.673018487663693,
|
|
"learning_rate": 1.210771204858263e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3613,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24460
|
|
},
|
|
{
|
|
"epoch": 0.9108335969924253,
|
|
"grad_norm": 8.046005132927798,
|
|
"learning_rate": 1.200804577954162e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3958,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.8671875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 24470
|
|
},
|
|
{
|
|
"epoch": 0.9112058215927491,
|
|
"grad_norm": 6.527869438405157,
|
|
"learning_rate": 1.1908781320937256e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24480
|
|
},
|
|
{
|
|
"epoch": 0.9115780461930729,
|
|
"grad_norm": 8.25432291845492,
|
|
"learning_rate": 1.1809918840361282e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3901,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24490
|
|
},
|
|
{
|
|
"epoch": 0.9119502707933967,
|
|
"grad_norm": 7.413804903487207,
|
|
"learning_rate": 1.171145850472688e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3781,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24500
|
|
},
|
|
{
|
|
"epoch": 0.9123224953937206,
|
|
"grad_norm": 8.122102879331281,
|
|
"learning_rate": 1.16134004802681e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 24510
|
|
},
|
|
{
|
|
"epoch": 0.9126947199940444,
|
|
"grad_norm": 6.553579050465894,
|
|
"learning_rate": 1.1515744932539929e-08,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3567,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24520
|
|
},
|
|
{
|
|
"epoch": 0.9130669445943682,
|
|
"grad_norm": 9.594175888257977,
|
|
"learning_rate": 1.1418492026417703e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4003,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24530
|
|
},
|
|
{
|
|
"epoch": 0.913439169194692,
|
|
"grad_norm": 8.90395593467981,
|
|
"learning_rate": 1.1321641926097137e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3529,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24540
|
|
},
|
|
{
|
|
"epoch": 0.9138113937950159,
|
|
"grad_norm": 8.259626575072058,
|
|
"learning_rate": 1.1225194795093623e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3893,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24550
|
|
},
|
|
{
|
|
"epoch": 0.9141836183953398,
|
|
"grad_norm": 7.5303375506795245,
|
|
"learning_rate": 1.1129150796242436e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3676,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24560
|
|
},
|
|
{
|
|
"epoch": 0.9145558429956636,
|
|
"grad_norm": 7.782874776100595,
|
|
"learning_rate": 1.1033510091698117e-08,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4008,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24570
|
|
},
|
|
{
|
|
"epoch": 0.9149280675959874,
|
|
"grad_norm": 9.341096969079205,
|
|
"learning_rate": 1.0938272842934304e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3704,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24580
|
|
},
|
|
{
|
|
"epoch": 0.9153002921963113,
|
|
"grad_norm": 8.549864859999074,
|
|
"learning_rate": 1.0843439210743572e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3995,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24590
|
|
},
|
|
{
|
|
"epoch": 0.9156725167966351,
|
|
"grad_norm": 8.207665400747823,
|
|
"learning_rate": 1.0749009355236898e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3885,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24600
|
|
},
|
|
{
|
|
"epoch": 0.9160447413969589,
|
|
"grad_norm": 8.402803733213904,
|
|
"learning_rate": 1.0654983435843645e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.384,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24610
|
|
},
|
|
{
|
|
"epoch": 0.9164169659972827,
|
|
"grad_norm": 8.96715349232335,
|
|
"learning_rate": 1.0561361611311188e-08,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3539,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 24620
|
|
},
|
|
{
|
|
"epoch": 0.9167891905976066,
|
|
"grad_norm": 7.933047363557474,
|
|
"learning_rate": 1.046814403970464e-08,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3915,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24630
|
|
},
|
|
{
|
|
"epoch": 0.9171614151979305,
|
|
"grad_norm": 7.680139589774546,
|
|
"learning_rate": 1.037533087840664e-08,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3826,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24640
|
|
},
|
|
{
|
|
"epoch": 0.9175336397982543,
|
|
"grad_norm": 8.688000765777499,
|
|
"learning_rate": 1.0282922284116951e-08,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24650
|
|
},
|
|
{
|
|
"epoch": 0.9179058643985781,
|
|
"grad_norm": 12.017177594281229,
|
|
"learning_rate": 1.0190918412852384e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.399,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24660
|
|
},
|
|
{
|
|
"epoch": 0.918278088998902,
|
|
"grad_norm": 8.052221548652287,
|
|
"learning_rate": 1.0099319419946406e-08,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3808,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24670
|
|
},
|
|
{
|
|
"epoch": 0.9186503135992258,
|
|
"grad_norm": 9.950919551868507,
|
|
"learning_rate": 1.0008125460048921e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3794,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24680
|
|
},
|
|
{
|
|
"epoch": 0.9190225381995496,
|
|
"grad_norm": 7.5343075612394435,
|
|
"learning_rate": 9.917336687125938e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24690
|
|
},
|
|
{
|
|
"epoch": 0.9193947627998734,
|
|
"grad_norm": 6.698183577310776,
|
|
"learning_rate": 9.826953254459508e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3795,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.96875,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24700
|
|
},
|
|
{
|
|
"epoch": 0.9197669874001972,
|
|
"grad_norm": 8.04771376429956,
|
|
"learning_rate": 9.736975314647205e-09,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3864,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24710
|
|
},
|
|
{
|
|
"epoch": 0.9201392120005211,
|
|
"grad_norm": 7.016386567764612,
|
|
"learning_rate": 9.647403019602068e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24720
|
|
},
|
|
{
|
|
"epoch": 0.920511436600845,
|
|
"grad_norm": 7.607169042502292,
|
|
"learning_rate": 9.558236520552265e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3787,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24730
|
|
},
|
|
{
|
|
"epoch": 0.9208836612011688,
|
|
"grad_norm": 7.16993301140087,
|
|
"learning_rate": 9.469475968040763e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.371,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24740
|
|
},
|
|
{
|
|
"epoch": 0.9212558858014926,
|
|
"grad_norm": 9.175764227138442,
|
|
"learning_rate": 9.381121511925243e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.4028,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24750
|
|
},
|
|
{
|
|
"epoch": 0.9216281104018165,
|
|
"grad_norm": 8.693800023349523,
|
|
"learning_rate": 9.293173301377772e-09,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24760
|
|
},
|
|
{
|
|
"epoch": 0.9220003350021403,
|
|
"grad_norm": 8.227242184689466,
|
|
"learning_rate": 9.205631484884485e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24770
|
|
},
|
|
{
|
|
"epoch": 0.9223725596024641,
|
|
"grad_norm": 8.675898496861459,
|
|
"learning_rate": 9.11849621024538e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3838,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24780
|
|
},
|
|
{
|
|
"epoch": 0.9227447842027879,
|
|
"grad_norm": 9.299724455890269,
|
|
"learning_rate": 9.031767624574083e-09,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3817,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24790
|
|
},
|
|
{
|
|
"epoch": 0.9231170088031118,
|
|
"grad_norm": 8.210261559554944,
|
|
"learning_rate": 8.945445874297659e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24800
|
|
},
|
|
{
|
|
"epoch": 0.9234892334034356,
|
|
"grad_norm": 8.152658507216266,
|
|
"learning_rate": 8.859531105156193e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3762,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.546875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 24810
|
|
},
|
|
{
|
|
"epoch": 0.9238614580037595,
|
|
"grad_norm": 7.170606784660298,
|
|
"learning_rate": 8.774023462202768e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 24820
|
|
},
|
|
{
|
|
"epoch": 0.9242336826040833,
|
|
"grad_norm": 8.048269813265286,
|
|
"learning_rate": 8.688923089802957e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3793,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24830
|
|
},
|
|
{
|
|
"epoch": 0.9246059072044072,
|
|
"grad_norm": 8.67052816830992,
|
|
"learning_rate": 8.604230131634832e-09,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 24840
|
|
},
|
|
{
|
|
"epoch": 0.924978131804731,
|
|
"grad_norm": 8.475443642733683,
|
|
"learning_rate": 8.519944730688594e-09,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3753,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24850
|
|
},
|
|
{
|
|
"epoch": 0.9253503564050548,
|
|
"grad_norm": 9.689006092506947,
|
|
"learning_rate": 8.436067029266358e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4073,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24860
|
|
},
|
|
{
|
|
"epoch": 0.9257225810053786,
|
|
"grad_norm": 8.152005674055406,
|
|
"learning_rate": 8.352597168981845e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3701,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24870
|
|
},
|
|
{
|
|
"epoch": 0.9260948056057025,
|
|
"grad_norm": 7.701441728875828,
|
|
"learning_rate": 8.269535290760271e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3697,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24880
|
|
},
|
|
{
|
|
"epoch": 0.9264670302060263,
|
|
"grad_norm": 7.3179891004814595,
|
|
"learning_rate": 8.18688153483793e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.375,
|
|
"step": 24890
|
|
},
|
|
{
|
|
"epoch": 0.9268392548063501,
|
|
"grad_norm": 8.220094953142276,
|
|
"learning_rate": 8.104636040762281e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3871,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24900
|
|
},
|
|
{
|
|
"epoch": 0.927211479406674,
|
|
"grad_norm": 7.322527195152629,
|
|
"learning_rate": 8.022798947391302e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3763,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 24910
|
|
},
|
|
{
|
|
"epoch": 0.9275837040069979,
|
|
"grad_norm": 8.593804189263999,
|
|
"learning_rate": 7.941370392893526e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3793,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 24920
|
|
},
|
|
{
|
|
"epoch": 0.9279559286073217,
|
|
"grad_norm": 7.000338122833403,
|
|
"learning_rate": 7.86035051474776e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.361,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 24930
|
|
},
|
|
{
|
|
"epoch": 0.9283281532076455,
|
|
"grad_norm": 7.2216467313188755,
|
|
"learning_rate": 7.779739449742723e-09,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3915,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24940
|
|
},
|
|
{
|
|
"epoch": 0.9287003778079693,
|
|
"grad_norm": 7.500521917104714,
|
|
"learning_rate": 7.699537333977101e-09,
|
|
"logits/chosen": -3.96875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 24950
|
|
},
|
|
{
|
|
"epoch": 0.9290726024082931,
|
|
"grad_norm": 8.966929070210888,
|
|
"learning_rate": 7.619744302858944e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3621,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.8046875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24960
|
|
},
|
|
{
|
|
"epoch": 0.929444827008617,
|
|
"grad_norm": 10.913713603314774,
|
|
"learning_rate": 7.540360491105763e-09,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4235,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.3125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 24970
|
|
},
|
|
{
|
|
"epoch": 0.9298170516089408,
|
|
"grad_norm": 7.552398917216502,
|
|
"learning_rate": 7.461386032744099e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3938,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24980
|
|
},
|
|
{
|
|
"epoch": 0.9301892762092646,
|
|
"grad_norm": 8.618566435560101,
|
|
"learning_rate": 7.382821061109407e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3877,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 24990
|
|
},
|
|
{
|
|
"epoch": 0.9305615008095885,
|
|
"grad_norm": 7.646477994740471,
|
|
"learning_rate": 7.304665708845803e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3779,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25000
|
|
},
|
|
{
|
|
"epoch": 0.9309337254099124,
|
|
"grad_norm": 7.732536490032479,
|
|
"learning_rate": 7.2269201079057375e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3783,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25010
|
|
},
|
|
{
|
|
"epoch": 0.9313059500102362,
|
|
"grad_norm": 8.128929788560097,
|
|
"learning_rate": 7.149584389549962e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25020
|
|
},
|
|
{
|
|
"epoch": 0.93167817461056,
|
|
"grad_norm": 7.92174609075386,
|
|
"learning_rate": 7.072658684347116e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3854,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25030
|
|
},
|
|
{
|
|
"epoch": 0.9320503992108838,
|
|
"grad_norm": 6.237690057356486,
|
|
"learning_rate": 6.996143122173753e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3564,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25040
|
|
},
|
|
{
|
|
"epoch": 0.9324226238112077,
|
|
"grad_norm": 6.880964369017207,
|
|
"learning_rate": 6.920037832213788e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3744,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25050
|
|
},
|
|
{
|
|
"epoch": 0.9327948484115315,
|
|
"grad_norm": 9.072517009352495,
|
|
"learning_rate": 6.8443429429585755e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3743,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25060
|
|
},
|
|
{
|
|
"epoch": 0.9331670730118553,
|
|
"grad_norm": 6.199769605737394,
|
|
"learning_rate": 6.769058582206555e-09,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3782,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 25070
|
|
},
|
|
{
|
|
"epoch": 0.9335392976121792,
|
|
"grad_norm": 9.50618039220734,
|
|
"learning_rate": 6.694184877062941e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.453125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3994,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25080
|
|
},
|
|
{
|
|
"epoch": 0.9339115222125031,
|
|
"grad_norm": 8.004458700772172,
|
|
"learning_rate": 6.619721953939866e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -1000.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3945,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -8.125,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25090
|
|
},
|
|
{
|
|
"epoch": 0.9342837468128269,
|
|
"grad_norm": 8.708436540968368,
|
|
"learning_rate": 6.545669938555654e-09,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3881,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25100
|
|
},
|
|
{
|
|
"epoch": 0.9346559714131507,
|
|
"grad_norm": 8.611938378660694,
|
|
"learning_rate": 6.4720289559351e-09,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25110
|
|
},
|
|
{
|
|
"epoch": 0.9350281960134745,
|
|
"grad_norm": 6.7756254393297874,
|
|
"learning_rate": 6.3987991304088885e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3466,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25120
|
|
},
|
|
{
|
|
"epoch": 0.9354004206137984,
|
|
"grad_norm": 7.610058315105068,
|
|
"learning_rate": 6.325980585613594e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3707,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25130
|
|
},
|
|
{
|
|
"epoch": 0.9357726452141222,
|
|
"grad_norm": 7.74588654322355,
|
|
"learning_rate": 6.2535734444914004e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3666,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 25140
|
|
},
|
|
{
|
|
"epoch": 0.936144869814446,
|
|
"grad_norm": 8.069527546471516,
|
|
"learning_rate": 6.181577829289936e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.38,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25150
|
|
},
|
|
{
|
|
"epoch": 0.9365170944147698,
|
|
"grad_norm": 8.7852059746855,
|
|
"learning_rate": 6.109993861561969e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4033,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25160
|
|
},
|
|
{
|
|
"epoch": 0.9368893190150938,
|
|
"grad_norm": 6.50557762738591,
|
|
"learning_rate": 6.0388216621652945e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3949,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25170
|
|
},
|
|
{
|
|
"epoch": 0.9372615436154176,
|
|
"grad_norm": 7.901286521733562,
|
|
"learning_rate": 5.968061351262599e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.3757,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 25180
|
|
},
|
|
{
|
|
"epoch": 0.9376337682157414,
|
|
"grad_norm": 8.244577897831062,
|
|
"learning_rate": 5.897713048320985e-09,
|
|
"logits/chosen": -3.609375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4029,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25190
|
|
},
|
|
{
|
|
"epoch": 0.9380059928160652,
|
|
"grad_norm": 7.467682930906925,
|
|
"learning_rate": 5.827776872112111e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3712,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25200
|
|
},
|
|
{
|
|
"epoch": 0.938378217416389,
|
|
"grad_norm": 6.991293981237631,
|
|
"learning_rate": 5.758252940711666e-09,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3671,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25210
|
|
},
|
|
{
|
|
"epoch": 0.9387504420167129,
|
|
"grad_norm": 7.119614202513316,
|
|
"learning_rate": 5.689141371499479e-09,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3883,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25220
|
|
},
|
|
{
|
|
"epoch": 0.9391226666170367,
|
|
"grad_norm": 6.044120884142725,
|
|
"learning_rate": 5.6204422811591546e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3511,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25230
|
|
},
|
|
{
|
|
"epoch": 0.9394948912173605,
|
|
"grad_norm": 8.647348076133104,
|
|
"learning_rate": 5.5521557856778035e-09,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25240
|
|
},
|
|
{
|
|
"epoch": 0.9398671158176843,
|
|
"grad_norm": 6.674179045856495,
|
|
"learning_rate": 5.484282000346007e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3819,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25250
|
|
},
|
|
{
|
|
"epoch": 0.9402393404180083,
|
|
"grad_norm": 7.266102997102009,
|
|
"learning_rate": 5.416821039757491e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -1024.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3759,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -8.1875,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 25260
|
|
},
|
|
{
|
|
"epoch": 0.9406115650183321,
|
|
"grad_norm": 6.195281467473668,
|
|
"learning_rate": 5.34977301780909e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3775,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25270
|
|
},
|
|
{
|
|
"epoch": 0.9409837896186559,
|
|
"grad_norm": 7.628597786279155,
|
|
"learning_rate": 5.283138047700392e-09,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.368,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25280
|
|
},
|
|
{
|
|
"epoch": 0.9413560142189797,
|
|
"grad_norm": 7.7298334508986,
|
|
"learning_rate": 5.216916241933683e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.3815,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 25290
|
|
},
|
|
{
|
|
"epoch": 0.9417282388193036,
|
|
"grad_norm": 8.8312986929598,
|
|
"learning_rate": 5.151107712313529e-09,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3777,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25300
|
|
},
|
|
{
|
|
"epoch": 0.9421004634196274,
|
|
"grad_norm": 8.206884388789925,
|
|
"learning_rate": 5.085712569946915e-09,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.4375,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25310
|
|
},
|
|
{
|
|
"epoch": 0.9424726880199512,
|
|
"grad_norm": 7.639550297538544,
|
|
"learning_rate": 5.020730925242828e-09,
|
|
"logits/chosen": -3.984375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3964,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25320
|
|
},
|
|
{
|
|
"epoch": 0.942844912620275,
|
|
"grad_norm": 8.539973927097225,
|
|
"learning_rate": 4.956162887912146e-09,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3898,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25330
|
|
},
|
|
{
|
|
"epoch": 0.9432171372205989,
|
|
"grad_norm": 9.030519803562058,
|
|
"learning_rate": 4.89200856696742e-09,
|
|
"logits/chosen": -4.03125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.4089,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25340
|
|
},
|
|
{
|
|
"epoch": 0.9435893618209228,
|
|
"grad_norm": 9.101739308547884,
|
|
"learning_rate": 4.828268070722674e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3782,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25350
|
|
},
|
|
{
|
|
"epoch": 0.9439615864212466,
|
|
"grad_norm": 9.080064303424589,
|
|
"learning_rate": 4.764941506793324e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3666,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 25360
|
|
},
|
|
{
|
|
"epoch": 0.9443338110215704,
|
|
"grad_norm": 7.1721141965024655,
|
|
"learning_rate": 4.702028982095901e-09,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.4061,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25370
|
|
},
|
|
{
|
|
"epoch": 0.9447060356218943,
|
|
"grad_norm": 7.206659789207086,
|
|
"learning_rate": 4.639530602847913e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4015,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25380
|
|
},
|
|
{
|
|
"epoch": 0.9450782602222181,
|
|
"grad_norm": 7.699175470799566,
|
|
"learning_rate": 4.57744647456762e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3685,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25390
|
|
},
|
|
{
|
|
"epoch": 0.9454504848225419,
|
|
"grad_norm": 8.18911099955022,
|
|
"learning_rate": 4.515776702073926e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25400
|
|
},
|
|
{
|
|
"epoch": 0.9458227094228657,
|
|
"grad_norm": 8.192140155456258,
|
|
"learning_rate": 4.454521389486099e-09,
|
|
"logits/chosen": -3.640625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3809,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25410
|
|
},
|
|
{
|
|
"epoch": 0.9461949340231895,
|
|
"grad_norm": 8.304497325448725,
|
|
"learning_rate": 4.393680640223718e-09,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3817,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25420
|
|
},
|
|
{
|
|
"epoch": 0.9465671586235135,
|
|
"grad_norm": 8.45804001964525,
|
|
"learning_rate": 4.333254557006477e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3706,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25430
|
|
},
|
|
{
|
|
"epoch": 0.9469393832238373,
|
|
"grad_norm": 7.201994651799432,
|
|
"learning_rate": 4.273243241853852e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3876,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25440
|
|
},
|
|
{
|
|
"epoch": 0.9473116078241611,
|
|
"grad_norm": 7.8202264056955135,
|
|
"learning_rate": 4.213646796085157e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.90625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25450
|
|
},
|
|
{
|
|
"epoch": 0.947683832424485,
|
|
"grad_norm": 9.240986557216605,
|
|
"learning_rate": 4.154465320319184e-09,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3692,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25460
|
|
},
|
|
{
|
|
"epoch": 0.9480560570248088,
|
|
"grad_norm": 7.163881057279578,
|
|
"learning_rate": 4.095698914474255e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3616,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25470
|
|
},
|
|
{
|
|
"epoch": 0.9484282816251326,
|
|
"grad_norm": 9.575734197774777,
|
|
"learning_rate": 4.037347677767728e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.401,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25480
|
|
},
|
|
{
|
|
"epoch": 0.9488005062254564,
|
|
"grad_norm": 8.584930471471967,
|
|
"learning_rate": 3.97941170871613e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3726,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25490
|
|
},
|
|
{
|
|
"epoch": 0.9491727308257802,
|
|
"grad_norm": 8.242687644929152,
|
|
"learning_rate": 3.921891105134884e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -1000.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3766,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25500
|
|
},
|
|
{
|
|
"epoch": 0.9495449554261041,
|
|
"grad_norm": 7.950155338016982,
|
|
"learning_rate": 3.864785964138112e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25510
|
|
},
|
|
{
|
|
"epoch": 0.949917180026428,
|
|
"grad_norm": 8.022078887595379,
|
|
"learning_rate": 3.80809638213847e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3636,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25520
|
|
},
|
|
{
|
|
"epoch": 0.9502894046267518,
|
|
"grad_norm": 7.620543755292294,
|
|
"learning_rate": 3.751822454847037e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3684,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25530
|
|
},
|
|
{
|
|
"epoch": 0.9506616292270756,
|
|
"grad_norm": 7.057994158121856,
|
|
"learning_rate": 3.6959642772730893e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3863,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25540
|
|
},
|
|
{
|
|
"epoch": 0.9510338538273995,
|
|
"grad_norm": 8.42999104994612,
|
|
"learning_rate": 3.6405219437240785e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.368,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25550
|
|
},
|
|
{
|
|
"epoch": 0.9514060784277233,
|
|
"grad_norm": 7.761289867539542,
|
|
"learning_rate": 3.5854955478052672e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3962,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 25560
|
|
},
|
|
{
|
|
"epoch": 0.9517783030280471,
|
|
"grad_norm": 7.509705477891007,
|
|
"learning_rate": 3.5308851824197296e-09,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.374,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25570
|
|
},
|
|
{
|
|
"epoch": 0.9521505276283709,
|
|
"grad_norm": 7.836024066864957,
|
|
"learning_rate": 3.476690939768101e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3767,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25580
|
|
},
|
|
{
|
|
"epoch": 0.9525227522286948,
|
|
"grad_norm": 7.976953392364282,
|
|
"learning_rate": 3.4229129113484968e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.39,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25590
|
|
},
|
|
{
|
|
"epoch": 0.9528949768290186,
|
|
"grad_norm": 6.9853021506721396,
|
|
"learning_rate": 3.3695511879562877e-09,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3691,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25600
|
|
},
|
|
{
|
|
"epoch": 0.9532672014293425,
|
|
"grad_norm": 8.78594864005694,
|
|
"learning_rate": 3.316605859684074e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3972,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25610
|
|
},
|
|
{
|
|
"epoch": 0.9536394260296663,
|
|
"grad_norm": 7.890092997553901,
|
|
"learning_rate": 3.264077015921268e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25620
|
|
},
|
|
{
|
|
"epoch": 0.9540116506299902,
|
|
"grad_norm": 7.351535705021673,
|
|
"learning_rate": 3.21196474535429e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3735,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 25630
|
|
},
|
|
{
|
|
"epoch": 0.954383875230314,
|
|
"grad_norm": 6.848652390705163,
|
|
"learning_rate": 3.1602691359661758e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3605,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25640
|
|
},
|
|
{
|
|
"epoch": 0.9547560998306378,
|
|
"grad_norm": 7.31096829576726,
|
|
"learning_rate": 3.108990275036444e-09,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3882,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25650
|
|
},
|
|
{
|
|
"epoch": 0.9551283244309616,
|
|
"grad_norm": 9.182565756535098,
|
|
"learning_rate": 3.0581282491411176e-09,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3849,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 25660
|
|
},
|
|
{
|
|
"epoch": 0.9555005490312855,
|
|
"grad_norm": 7.409904144952999,
|
|
"learning_rate": 3.0076831441523677e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3806,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25670
|
|
},
|
|
{
|
|
"epoch": 0.9558727736316093,
|
|
"grad_norm": 8.938718675523969,
|
|
"learning_rate": 2.957655045238455e-09,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3923,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5078125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25680
|
|
},
|
|
{
|
|
"epoch": 0.9562449982319331,
|
|
"grad_norm": 9.723696579686314,
|
|
"learning_rate": 2.9080440368636773e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4153,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.6171875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25690
|
|
},
|
|
{
|
|
"epoch": 0.956617222832257,
|
|
"grad_norm": 8.04076944223611,
|
|
"learning_rate": 2.8588502027881156e-09,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3716,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 25700
|
|
},
|
|
{
|
|
"epoch": 0.9569894474325809,
|
|
"grad_norm": 9.634659005283998,
|
|
"learning_rate": 2.8100736260674442e-09,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3803,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25710
|
|
},
|
|
{
|
|
"epoch": 0.9573616720329047,
|
|
"grad_norm": 6.901137941390378,
|
|
"learning_rate": 2.761714389052955e-09,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3828,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25720
|
|
},
|
|
{
|
|
"epoch": 0.9577338966332285,
|
|
"grad_norm": 7.3490690562903564,
|
|
"learning_rate": 2.713772573391282e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25730
|
|
},
|
|
{
|
|
"epoch": 0.9581061212335523,
|
|
"grad_norm": 8.00106848232399,
|
|
"learning_rate": 2.6662482600243452e-09,
|
|
"logits/chosen": -3.625,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4068,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25740
|
|
},
|
|
{
|
|
"epoch": 0.9584783458338761,
|
|
"grad_norm": 6.983323090970681,
|
|
"learning_rate": 2.619141529189184e-09,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3952,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25750
|
|
},
|
|
{
|
|
"epoch": 0.9588505704342,
|
|
"grad_norm": 8.016712851585822,
|
|
"learning_rate": 2.572452460417762e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3935,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25760
|
|
},
|
|
{
|
|
"epoch": 0.9592227950345238,
|
|
"grad_norm": 6.784253680078157,
|
|
"learning_rate": 2.526181132536942e-09,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3662,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25770
|
|
},
|
|
{
|
|
"epoch": 0.9595950196348476,
|
|
"grad_norm": 7.511516161088099,
|
|
"learning_rate": 2.480327623668288e-09,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3742,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25780
|
|
},
|
|
{
|
|
"epoch": 0.9599672442351715,
|
|
"grad_norm": 7.705818125137246,
|
|
"learning_rate": 2.434892011227929e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -988.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3846,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25790
|
|
},
|
|
{
|
|
"epoch": 0.9603394688354954,
|
|
"grad_norm": 9.626247526417059,
|
|
"learning_rate": 2.389874371926448e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4107,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 25800
|
|
},
|
|
{
|
|
"epoch": 0.9607116934358192,
|
|
"grad_norm": 7.614119081193586,
|
|
"learning_rate": 2.345274781768769e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3677,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25810
|
|
},
|
|
{
|
|
"epoch": 0.961083918036143,
|
|
"grad_norm": 7.710932115104765,
|
|
"learning_rate": 2.3010933160539927e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4023,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 25820
|
|
},
|
|
{
|
|
"epoch": 0.9614561426364668,
|
|
"grad_norm": 9.135037333147595,
|
|
"learning_rate": 2.2573300493752835e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25830
|
|
},
|
|
{
|
|
"epoch": 0.9618283672367907,
|
|
"grad_norm": 7.394113627693234,
|
|
"learning_rate": 2.213985055619788e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3785,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25840
|
|
},
|
|
{
|
|
"epoch": 0.9622005918371145,
|
|
"grad_norm": 8.499242477723133,
|
|
"learning_rate": 2.171058407968357e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3953,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25850
|
|
},
|
|
{
|
|
"epoch": 0.9625728164374383,
|
|
"grad_norm": 7.266419948567692,
|
|
"learning_rate": 2.128550178895655e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.375,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 25860
|
|
},
|
|
{
|
|
"epoch": 0.9629450410377622,
|
|
"grad_norm": 7.837807789912403,
|
|
"learning_rate": 2.086460440169857e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.5,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3914,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25870
|
|
},
|
|
{
|
|
"epoch": 0.9633172656380861,
|
|
"grad_norm": 11.412423144164832,
|
|
"learning_rate": 2.044789262852592e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4086,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25880
|
|
},
|
|
{
|
|
"epoch": 0.9636894902384099,
|
|
"grad_norm": 7.196527551488307,
|
|
"learning_rate": 2.003536717298804e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.36,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25890
|
|
},
|
|
{
|
|
"epoch": 0.9640617148387337,
|
|
"grad_norm": 8.546640472547097,
|
|
"learning_rate": 1.9627028731566685e-09,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.4005,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 25900
|
|
},
|
|
{
|
|
"epoch": 0.9644339394390575,
|
|
"grad_norm": 8.873598572493062,
|
|
"learning_rate": 1.922287799367456e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 25910
|
|
},
|
|
{
|
|
"epoch": 0.9648061640393814,
|
|
"grad_norm": 6.730252328323946,
|
|
"learning_rate": 1.882291564165389e-09,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3553,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25920
|
|
},
|
|
{
|
|
"epoch": 0.9651783886397052,
|
|
"grad_norm": 9.432094421897425,
|
|
"learning_rate": 1.8427142350775638e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.4011,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 25930
|
|
},
|
|
{
|
|
"epoch": 0.965550613240029,
|
|
"grad_norm": 8.063041412460393,
|
|
"learning_rate": 1.803555878923807e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3909,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25940
|
|
},
|
|
{
|
|
"epoch": 0.9659228378403528,
|
|
"grad_norm": 7.856947739138653,
|
|
"learning_rate": 1.7648165618166233e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3907,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25950
|
|
},
|
|
{
|
|
"epoch": 0.9662950624406768,
|
|
"grad_norm": 7.149228565332752,
|
|
"learning_rate": 1.726496349160944e-09,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -912.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3409,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -7.34375,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25960
|
|
},
|
|
{
|
|
"epoch": 0.9666672870410006,
|
|
"grad_norm": 8.637886147027949,
|
|
"learning_rate": 1.688595305654239e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3854,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 25970
|
|
},
|
|
{
|
|
"epoch": 0.9670395116413244,
|
|
"grad_norm": 6.803464149957781,
|
|
"learning_rate": 1.651113495286155e-09,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3905,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 25980
|
|
},
|
|
{
|
|
"epoch": 0.9674117362416482,
|
|
"grad_norm": 8.53945800231934,
|
|
"learning_rate": 1.6140509813386273e-09,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3868,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 25990
|
|
},
|
|
{
|
|
"epoch": 0.967783960841972,
|
|
"grad_norm": 8.034083834246642,
|
|
"learning_rate": 1.5774078263856571e-09,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3869,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26000
|
|
},
|
|
{
|
|
"epoch": 0.9681561854422959,
|
|
"grad_norm": 6.689395084208392,
|
|
"learning_rate": 1.5411840922931184e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3638,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26010
|
|
},
|
|
{
|
|
"epoch": 0.9685284100426197,
|
|
"grad_norm": 8.928521789523673,
|
|
"learning_rate": 1.5053798402189788e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3893,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26020
|
|
},
|
|
{
|
|
"epoch": 0.9689006346429435,
|
|
"grad_norm": 7.876486006792816,
|
|
"learning_rate": 1.4699951306127722e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -996.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3634,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -8.0,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 26030
|
|
},
|
|
{
|
|
"epoch": 0.9692728592432673,
|
|
"grad_norm": 8.624619581012555,
|
|
"learning_rate": 1.4350300232158218e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3935,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26040
|
|
},
|
|
{
|
|
"epoch": 0.9696450838435913,
|
|
"grad_norm": 8.442683954767702,
|
|
"learning_rate": 1.400484577060962e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3776,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26050
|
|
},
|
|
{
|
|
"epoch": 0.9700173084439151,
|
|
"grad_norm": 6.482439286903222,
|
|
"learning_rate": 1.3663588504725098e-09,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3732,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 26060
|
|
},
|
|
{
|
|
"epoch": 0.9703895330442389,
|
|
"grad_norm": 9.531796726134484,
|
|
"learning_rate": 1.3326529010662112e-09,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3807,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.46875,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26070
|
|
},
|
|
{
|
|
"epoch": 0.9707617576445627,
|
|
"grad_norm": 8.857674777296513,
|
|
"learning_rate": 1.2993667857489898e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3614,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26080
|
|
},
|
|
{
|
|
"epoch": 0.9711339822448866,
|
|
"grad_norm": 7.014517356456569,
|
|
"learning_rate": 1.2665005607190581e-09,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3806,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 26090
|
|
},
|
|
{
|
|
"epoch": 0.9715062068452104,
|
|
"grad_norm": 7.6096605678365155,
|
|
"learning_rate": 1.2340542814655853e-09,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3685,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26100
|
|
},
|
|
{
|
|
"epoch": 0.9718784314455342,
|
|
"grad_norm": 8.838364450015678,
|
|
"learning_rate": 1.2020280027688622e-09,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3909,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26110
|
|
},
|
|
{
|
|
"epoch": 0.972250656045858,
|
|
"grad_norm": 7.031397615856246,
|
|
"learning_rate": 1.1704217786999703e-09,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.46875,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3689,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26120
|
|
},
|
|
{
|
|
"epoch": 0.9726228806461819,
|
|
"grad_norm": 8.028911684431462,
|
|
"learning_rate": 1.139235662620891e-09,
|
|
"logits/chosen": -4.0,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -996.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.4078,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.96875,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26130
|
|
},
|
|
{
|
|
"epoch": 0.9729951052465058,
|
|
"grad_norm": 8.096150377178802,
|
|
"learning_rate": 1.1084697071842008e-09,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3767,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26140
|
|
},
|
|
{
|
|
"epoch": 0.9733673298468296,
|
|
"grad_norm": 6.9690236203072295,
|
|
"learning_rate": 1.0781239643332384e-09,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3756,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 26150
|
|
},
|
|
{
|
|
"epoch": 0.9737395544471534,
|
|
"grad_norm": 9.170186623419715,
|
|
"learning_rate": 1.0481984853017988e-09,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3744,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.6328125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26160
|
|
},
|
|
{
|
|
"epoch": 0.9741117790474773,
|
|
"grad_norm": 8.2020919164888,
|
|
"learning_rate": 1.0186933206141612e-09,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26170
|
|
},
|
|
{
|
|
"epoch": 0.9744840036478011,
|
|
"grad_norm": 7.021199260885473,
|
|
"learning_rate": 9.89608520084978e-10,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -1000.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 26180
|
|
},
|
|
{
|
|
"epoch": 0.9748562282481249,
|
|
"grad_norm": 8.601732580230633,
|
|
"learning_rate": 9.609441328191082e-10,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3859,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26190
|
|
},
|
|
{
|
|
"epoch": 0.9752284528484487,
|
|
"grad_norm": 7.879305377661017,
|
|
"learning_rate": 9.327002072117563e-10,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26200
|
|
},
|
|
{
|
|
"epoch": 0.9756006774487725,
|
|
"grad_norm": 7.495771705507001,
|
|
"learning_rate": 9.048767909480837e-10,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.515625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3734,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26210
|
|
},
|
|
{
|
|
"epoch": 0.9759729020490965,
|
|
"grad_norm": 8.97599450920641,
|
|
"learning_rate": 8.774739310034585e-10,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3623,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26220
|
|
},
|
|
{
|
|
"epoch": 0.9763451266494203,
|
|
"grad_norm": 7.245703986422246,
|
|
"learning_rate": 8.504916736430667e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.546875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4035,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.9296875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 26230
|
|
},
|
|
{
|
|
"epoch": 0.9767173512497441,
|
|
"grad_norm": 7.369988705394834,
|
|
"learning_rate": 8.239300644220237e-10,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -1004.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 26240
|
|
},
|
|
{
|
|
"epoch": 0.977089575850068,
|
|
"grad_norm": 6.693826961840806,
|
|
"learning_rate": 7.977891481852906e-10,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3994,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26250
|
|
},
|
|
{
|
|
"epoch": 0.9774618004503918,
|
|
"grad_norm": 8.493778981930056,
|
|
"learning_rate": 7.720689690674798e-10,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26260
|
|
},
|
|
{
|
|
"epoch": 0.9778340250507156,
|
|
"grad_norm": 8.683745925203684,
|
|
"learning_rate": 7.467695704929666e-10,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3676,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26270
|
|
},
|
|
{
|
|
"epoch": 0.9782062496510394,
|
|
"grad_norm": 8.510203289586169,
|
|
"learning_rate": 7.218909951755836e-10,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26280
|
|
},
|
|
{
|
|
"epoch": 0.9785784742513632,
|
|
"grad_norm": 8.422073908586597,
|
|
"learning_rate": 6.974332851187592e-10,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3966,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26290
|
|
},
|
|
{
|
|
"epoch": 0.9789506988516871,
|
|
"grad_norm": 7.934683815889488,
|
|
"learning_rate": 6.733964816153237e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3664,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 26300
|
|
},
|
|
{
|
|
"epoch": 0.979322923452011,
|
|
"grad_norm": 8.52846756729006,
|
|
"learning_rate": 6.497806252474814e-10,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -920.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 26310
|
|
},
|
|
{
|
|
"epoch": 0.9796951480523348,
|
|
"grad_norm": 7.500129875611216,
|
|
"learning_rate": 6.265857558867271e-10,
|
|
"logits/chosen": -3.65625,
|
|
"logits/rejected": -3.421875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3686,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26320
|
|
},
|
|
{
|
|
"epoch": 0.9800673726526586,
|
|
"grad_norm": 7.033432245711564,
|
|
"learning_rate": 6.038119126938191e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 26330
|
|
},
|
|
{
|
|
"epoch": 0.9804395972529825,
|
|
"grad_norm": 8.099975270251251,
|
|
"learning_rate": 5.814591341186392e-10,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3839,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26340
|
|
},
|
|
{
|
|
"epoch": 0.9808118218533063,
|
|
"grad_norm": 8.726221195912677,
|
|
"learning_rate": 5.595274579002219e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3818,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.828125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26350
|
|
},
|
|
{
|
|
"epoch": 0.9811840464536301,
|
|
"grad_norm": 9.928252400465533,
|
|
"learning_rate": 5.380169210666141e-10,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4125,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26360
|
|
},
|
|
{
|
|
"epoch": 0.9815562710539539,
|
|
"grad_norm": 8.144552414545608,
|
|
"learning_rate": 5.169275599347933e-10,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -1004.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.3916,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 26370
|
|
},
|
|
{
|
|
"epoch": 0.9819284956542778,
|
|
"grad_norm": 7.055860823037098,
|
|
"learning_rate": 4.96259410110722e-10,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3515,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26380
|
|
},
|
|
{
|
|
"epoch": 0.9823007202546016,
|
|
"grad_norm": 9.379110287297957,
|
|
"learning_rate": 4.760125064891818e-10,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -992.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26390
|
|
},
|
|
{
|
|
"epoch": 0.9826729448549255,
|
|
"grad_norm": 8.825731919272068,
|
|
"learning_rate": 4.561868832537452e-10,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26400
|
|
},
|
|
{
|
|
"epoch": 0.9830451694552493,
|
|
"grad_norm": 8.113303563976592,
|
|
"learning_rate": 4.367825738767206e-10,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -916.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3798,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 26410
|
|
},
|
|
{
|
|
"epoch": 0.9834173940555732,
|
|
"grad_norm": 6.203117194528164,
|
|
"learning_rate": 4.1779961111915176e-10,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 26420
|
|
},
|
|
{
|
|
"epoch": 0.983789618655897,
|
|
"grad_norm": 7.3037570614994225,
|
|
"learning_rate": 3.9923802703059616e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3735,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26430
|
|
},
|
|
{
|
|
"epoch": 0.9841618432562208,
|
|
"grad_norm": 7.349666964821333,
|
|
"learning_rate": 3.810978529493192e-10,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3739,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26440
|
|
},
|
|
{
|
|
"epoch": 0.9845340678565446,
|
|
"grad_norm": 9.394718977643159,
|
|
"learning_rate": 3.633791195019886e-10,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3889,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26450
|
|
},
|
|
{
|
|
"epoch": 0.9849062924568684,
|
|
"grad_norm": 8.222554499994736,
|
|
"learning_rate": 3.4608185660378595e-10,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3697,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.5625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26460
|
|
},
|
|
{
|
|
"epoch": 0.9852785170571923,
|
|
"grad_norm": 8.970557809887021,
|
|
"learning_rate": 3.292060934582952e-10,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4062,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 26470
|
|
},
|
|
{
|
|
"epoch": 0.9856507416575161,
|
|
"grad_norm": 7.660644951796844,
|
|
"learning_rate": 3.127518585575306e-10,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3822,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26480
|
|
},
|
|
{
|
|
"epoch": 0.98602296625784,
|
|
"grad_norm": 8.100154283810252,
|
|
"learning_rate": 2.9671917968171476e-10,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26490
|
|
},
|
|
{
|
|
"epoch": 0.9863951908581639,
|
|
"grad_norm": 7.304844776160379,
|
|
"learning_rate": 2.8110808389944506e-10,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3845,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 26500
|
|
},
|
|
{
|
|
"epoch": 0.9867674154584877,
|
|
"grad_norm": 7.058249139918956,
|
|
"learning_rate": 2.6591859756749934e-10,
|
|
"logits/chosen": -3.921875,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26510
|
|
},
|
|
{
|
|
"epoch": 0.9871396400588115,
|
|
"grad_norm": 9.624837392492937,
|
|
"learning_rate": 2.511507463307805e-10,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26520
|
|
},
|
|
{
|
|
"epoch": 0.9875118646591353,
|
|
"grad_norm": 7.261787552732307,
|
|
"learning_rate": 2.3680455512242736e-10,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3932,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26530
|
|
},
|
|
{
|
|
"epoch": 0.9878840892594591,
|
|
"grad_norm": 7.417981145540084,
|
|
"learning_rate": 2.2288004816364836e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -996.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3787,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.90625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26540
|
|
},
|
|
{
|
|
"epoch": 0.988256313859783,
|
|
"grad_norm": 6.71292939176362,
|
|
"learning_rate": 2.0937724896369358e-10,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.7265625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26550
|
|
},
|
|
{
|
|
"epoch": 0.9886285384601068,
|
|
"grad_norm": 8.317718776409816,
|
|
"learning_rate": 1.9629618031977159e-10,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3665,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26560
|
|
},
|
|
{
|
|
"epoch": 0.9890007630604306,
|
|
"grad_norm": 7.053556006564124,
|
|
"learning_rate": 1.8363686431718817e-10,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -972.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.366,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26570
|
|
},
|
|
{
|
|
"epoch": 0.9893729876607545,
|
|
"grad_norm": 9.23385581437373,
|
|
"learning_rate": 1.713993223290966e-10,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3688,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.84375,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 26580
|
|
},
|
|
{
|
|
"epoch": 0.9897452122610784,
|
|
"grad_norm": 8.314701860199557,
|
|
"learning_rate": 1.5958357501658082e-10,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.53125,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3615,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 26590
|
|
},
|
|
{
|
|
"epoch": 0.9901174368614022,
|
|
"grad_norm": 7.725841886499732,
|
|
"learning_rate": 1.4818964232859997e-10,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.361,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26600
|
|
},
|
|
{
|
|
"epoch": 0.990489661461726,
|
|
"grad_norm": 7.796435830384096,
|
|
"learning_rate": 1.3721754350196068e-10,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -924.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.3747,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 26610
|
|
},
|
|
{
|
|
"epoch": 0.9908618860620498,
|
|
"grad_norm": 7.350345197009885,
|
|
"learning_rate": 1.2666729706120593e-10,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.3831,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26620
|
|
},
|
|
{
|
|
"epoch": 0.9912341106623737,
|
|
"grad_norm": 8.828404979561103,
|
|
"learning_rate": 1.1653892081875393e-10,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.4014,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 26630
|
|
},
|
|
{
|
|
"epoch": 0.9916063352626975,
|
|
"grad_norm": 7.312678909739162,
|
|
"learning_rate": 1.0683243187467605e-10,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3622,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26640
|
|
},
|
|
{
|
|
"epoch": 0.9919785598630213,
|
|
"grad_norm": 8.33282303234302,
|
|
"learning_rate": 9.754784661680781e-11,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3802,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26650
|
|
},
|
|
{
|
|
"epoch": 0.9923507844633452,
|
|
"grad_norm": 7.7355287942966635,
|
|
"learning_rate": 8.86851807206379e-11,
|
|
"logits/chosen": -3.671875,
|
|
"logits/rejected": -3.484375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3792,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.71875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26660
|
|
},
|
|
{
|
|
"epoch": 0.9927230090636691,
|
|
"grad_norm": 8.331555141151071,
|
|
"learning_rate": 8.024444914933592e-11,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -928.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3823,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -7.40625,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26670
|
|
},
|
|
{
|
|
"epoch": 0.9930952336639929,
|
|
"grad_norm": 6.888738767669416,
|
|
"learning_rate": 7.222566615369685e-11,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.3831,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.4765625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 26680
|
|
},
|
|
{
|
|
"epoch": 0.9934674582643167,
|
|
"grad_norm": 8.176008013265742,
|
|
"learning_rate": 6.462884527208557e-11,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.734375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 26690
|
|
},
|
|
{
|
|
"epoch": 0.9938396828646405,
|
|
"grad_norm": 8.115720285827337,
|
|
"learning_rate": 5.745399933054784e-11,
|
|
"logits/chosen": -3.90625,
|
|
"logits/rejected": -3.671875,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3888,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26700
|
|
},
|
|
{
|
|
"epoch": 0.9942119074649644,
|
|
"grad_norm": 7.890797651154026,
|
|
"learning_rate": 5.0701140442588333e-11,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3612,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -7.6875,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26710
|
|
},
|
|
{
|
|
"epoch": 0.9945841320652882,
|
|
"grad_norm": 10.230403930874214,
|
|
"learning_rate": 4.437028000933707e-11,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.397,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 26720
|
|
},
|
|
{
|
|
"epoch": 0.994956356665612,
|
|
"grad_norm": 8.550979464025039,
|
|
"learning_rate": 3.846142871938296e-11,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.640625,
|
|
"logps/chosen": -940.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.4121,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -7.625,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 26730
|
|
},
|
|
{
|
|
"epoch": 0.9953285812659358,
|
|
"grad_norm": 7.527867131169457,
|
|
"learning_rate": 3.2974596548884795e-11,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -956.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3609,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.59375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26740
|
|
},
|
|
{
|
|
"epoch": 0.9957008058662598,
|
|
"grad_norm": 7.341687348552873,
|
|
"learning_rate": 2.790979276143246e-11,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.71875,
|
|
"logps/chosen": -984.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.3767,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.875,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26750
|
|
},
|
|
{
|
|
"epoch": 0.9960730304665836,
|
|
"grad_norm": 8.364728994355168,
|
|
"learning_rate": 2.3267025908130232e-11,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -936.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3874,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -7.4375,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26760
|
|
},
|
|
{
|
|
"epoch": 0.9964452550669074,
|
|
"grad_norm": 7.5238827189802056,
|
|
"learning_rate": 1.9046303827569e-11,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.609375,
|
|
"logps/chosen": -932.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3651,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.53125,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26770
|
|
},
|
|
{
|
|
"epoch": 0.9968174796672312,
|
|
"grad_norm": 7.528501034597796,
|
|
"learning_rate": 1.524763364565973e-11,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.5625,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.3758,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -7.78125,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26780
|
|
},
|
|
{
|
|
"epoch": 0.997189704267555,
|
|
"grad_norm": 7.550658488940014,
|
|
"learning_rate": 1.1871021775911039e-11,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.65625,
|
|
"logps/chosen": -976.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.4016,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26790
|
|
},
|
|
{
|
|
"epoch": 0.9975619288678789,
|
|
"grad_norm": 7.222129276789185,
|
|
"learning_rate": 8.916473919151624e-12,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.578125,
|
|
"logps/chosen": -948.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3696,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -7.65625,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26800
|
|
},
|
|
{
|
|
"epoch": 0.9979341534682027,
|
|
"grad_norm": 6.863301944839253,
|
|
"learning_rate": 6.38399506364129e-12,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.59375,
|
|
"logps/chosen": -952.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.3666,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 26810
|
|
},
|
|
{
|
|
"epoch": 0.9983063780685265,
|
|
"grad_norm": 8.646743667031297,
|
|
"learning_rate": 4.27358948507095e-12,
|
|
"logits/chosen": -3.6875,
|
|
"logits/rejected": -3.703125,
|
|
"logps/chosen": -968.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.3861,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.5625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 26820
|
|
},
|
|
{
|
|
"epoch": 0.9986786026688503,
|
|
"grad_norm": 7.804936344486104,
|
|
"learning_rate": 2.5852607465071118e-12,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -964.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.3913,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.8125,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26830
|
|
},
|
|
{
|
|
"epoch": 0.9990508272691743,
|
|
"grad_norm": 8.133430788662688,
|
|
"learning_rate": 1.3190116984196365e-12,
|
|
"logits/chosen": -3.9375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -960.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.3836,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -7.75,
|
|
"rewards/margins": 1.671875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 26840
|
|
},
|
|
{
|
|
"epoch": 0.9994230518694981,
|
|
"grad_norm": 9.3554182754693,
|
|
"learning_rate": 4.748444786539796e-13,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -980.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.9375,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 26850
|
|
},
|
|
{
|
|
"epoch": 0.9997952764698219,
|
|
"grad_norm": 8.95478383841915,
|
|
"learning_rate": 5.2760512486704765e-14,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.625,
|
|
"logps/chosen": -944.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -7.5,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 26860
|
|
},
|
|
{
|
|
"epoch": 0.9999813887699838,
|
|
"step": 26865,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.0,
|
|
"train_runtime": 37.9451,
|
|
"train_samples_per_second": 90625.095,
|
|
"train_steps_per_second": 707.997
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 26865,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 10000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|