20865 lines
614 KiB
JSON
20865 lines
614 KiB
JSON
{
|
|
"best_metric": 0.21195410192012787,
|
|
"best_model_checkpoint": "models/qwen2.5-3b-dpo-finegrained/checkpoint-10000",
|
|
"epoch": 1.0,
|
|
"eval_steps": 5000,
|
|
"global_step": 13853,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 7.218652999350322e-05,
|
|
"grad_norm": 1.852354340577762,
|
|
"learning_rate": 3.6075036075036073e-10,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -1.6015625,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0007218652999350321,
|
|
"grad_norm": 1.998199384338708,
|
|
"learning_rate": 3.6075036075036073e-09,
|
|
"logits/chosen": -2.265625,
|
|
"logits/rejected": -1.953125,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -143.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.2638888955116272,
|
|
"rewards/chosen": 0.000766754150390625,
|
|
"rewards/margins": 0.000606536865234375,
|
|
"rewards/rejected": 0.000156402587890625,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0014437305998700643,
|
|
"grad_norm": 1.859902408963682,
|
|
"learning_rate": 7.215007215007215e-09,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.001129150390625,
|
|
"rewards/margins": 0.00061798095703125,
|
|
"rewards/rejected": 0.000507354736328125,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0021655958998050965,
|
|
"grad_norm": 1.800945636929482,
|
|
"learning_rate": 1.0822510822510822e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -148.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.3375000059604645,
|
|
"rewards/chosen": 0.000640869140625,
|
|
"rewards/margins": -0.0003757476806640625,
|
|
"rewards/rejected": 0.00101470947265625,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.0028874611997401285,
|
|
"grad_norm": 1.9512697515276394,
|
|
"learning_rate": 1.443001443001443e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.2874999940395355,
|
|
"rewards/chosen": 0.00010967254638671875,
|
|
"rewards/margins": 0.0003910064697265625,
|
|
"rewards/rejected": -0.00028228759765625,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.0036093264996751606,
|
|
"grad_norm": 1.9685077450218404,
|
|
"learning_rate": 1.8037518037518035e-08,
|
|
"logits/chosen": -2.21875,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -143.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.3187499940395355,
|
|
"rewards/chosen": 9.441375732421875e-05,
|
|
"rewards/margins": 0.0001239776611328125,
|
|
"rewards/rejected": -3.0159950256347656e-05,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.004331191799610193,
|
|
"grad_norm": 1.9944209672205195,
|
|
"learning_rate": 2.1645021645021644e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -160.0,
|
|
"logps/rejected": -136.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": -0.000232696533203125,
|
|
"rewards/margins": 0.00102996826171875,
|
|
"rewards/rejected": -0.0012664794921875,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.005053057099545225,
|
|
"grad_norm": 1.8602055997261977,
|
|
"learning_rate": 2.525252525252525e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.8984375,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.3375000059604645,
|
|
"rewards/chosen": 0.000438690185546875,
|
|
"rewards/margins": 0.00054931640625,
|
|
"rewards/rejected": -0.00010919570922851562,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.005774922399480257,
|
|
"grad_norm": 1.7129972696287459,
|
|
"learning_rate": 2.886002886002886e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -170.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.000579833984375,
|
|
"rewards/margins": 0.0004520416259765625,
|
|
"rewards/rejected": 0.0001239776611328125,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.006496787699415289,
|
|
"grad_norm": 1.8246478253492482,
|
|
"learning_rate": 3.246753246753246e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -178.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 0.000469207763671875,
|
|
"rewards/margins": 0.0001888275146484375,
|
|
"rewards/rejected": 0.0002803802490234375,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.007218652999350321,
|
|
"grad_norm": 1.7657315202986226,
|
|
"learning_rate": 3.607503607503607e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3062500059604645,
|
|
"rewards/chosen": 0.000797271728515625,
|
|
"rewards/margins": -0.0003910064697265625,
|
|
"rewards/rejected": 0.001190185546875,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.007940518299285354,
|
|
"grad_norm": 2.0123631368082844,
|
|
"learning_rate": 3.968253968253968e-08,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.9296875,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -140.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.35624998807907104,
|
|
"rewards/chosen": 0.000186920166015625,
|
|
"rewards/margins": 0.00018787384033203125,
|
|
"rewards/rejected": 1.1473894119262695e-06,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.008662383599220386,
|
|
"grad_norm": 1.9667654175700489,
|
|
"learning_rate": 4.329004329004329e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.33125001192092896,
|
|
"rewards/chosen": 0.00154876708984375,
|
|
"rewards/margins": -0.00103759765625,
|
|
"rewards/rejected": 0.0025787353515625,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.009384248899155418,
|
|
"grad_norm": 1.7516436962452608,
|
|
"learning_rate": 4.68975468975469e-08,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.4312500059604645,
|
|
"rewards/chosen": 0.0027313232421875,
|
|
"rewards/margins": 0.00164031982421875,
|
|
"rewards/rejected": 0.0010986328125,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.01010611419909045,
|
|
"grad_norm": 2.0629971445302435,
|
|
"learning_rate": 5.05050505050505e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -167.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6921,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.0018768310546875,
|
|
"rewards/margins": 0.000438690185546875,
|
|
"rewards/rejected": 0.00144195556640625,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.010827979499025482,
|
|
"grad_norm": 2.0455409873804205,
|
|
"learning_rate": 5.411255411255411e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.40625,
|
|
"rewards/chosen": 0.001739501953125,
|
|
"rewards/margins": 5.435943603515625e-05,
|
|
"rewards/rejected": 0.001678466796875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.011549844798960514,
|
|
"grad_norm": 2.003008491192784,
|
|
"learning_rate": 5.772005772005772e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -1.8359375,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.6919,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.0026702880859375,
|
|
"rewards/margins": 0.00093841552734375,
|
|
"rewards/rejected": 0.00173187255859375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.012271710098895546,
|
|
"grad_norm": 1.8970983946398985,
|
|
"learning_rate": 6.132756132756133e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.9921875,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6915,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.00445556640625,
|
|
"rewards/margins": 0.00396728515625,
|
|
"rewards/rejected": 0.0004787445068359375,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.012993575398830578,
|
|
"grad_norm": 1.9000812663628288,
|
|
"learning_rate": 6.493506493506492e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -184.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6913,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.004791259765625,
|
|
"rewards/margins": 0.00274658203125,
|
|
"rewards/rejected": 0.0020294189453125,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.01371544069876561,
|
|
"grad_norm": 1.7694192875136483,
|
|
"learning_rate": 6.854256854256854e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.007598876953125,
|
|
"rewards/margins": 0.003387451171875,
|
|
"rewards/rejected": 0.00421142578125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.014437305998700642,
|
|
"grad_norm": 1.8022914272788901,
|
|
"learning_rate": 7.215007215007214e-08,
|
|
"logits/chosen": -2.296875,
|
|
"logits/rejected": -1.890625,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6907,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": 0.007354736328125,
|
|
"rewards/margins": 0.005584716796875,
|
|
"rewards/rejected": 0.0017547607421875,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.015159171298635674,
|
|
"grad_norm": 1.7754509638333658,
|
|
"learning_rate": 7.575757575757576e-08,
|
|
"logits/chosen": -2.328125,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6908,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.007537841796875,
|
|
"rewards/margins": 0.0034942626953125,
|
|
"rewards/rejected": 0.0040283203125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.015881036598570708,
|
|
"grad_norm": 1.8713087698233093,
|
|
"learning_rate": 7.936507936507936e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -1.8984375,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.00860595703125,
|
|
"rewards/margins": 0.00469970703125,
|
|
"rewards/rejected": 0.00390625,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.01660290189850574,
|
|
"grad_norm": 1.7352700913418746,
|
|
"learning_rate": 8.297258297258297e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -150.0,
|
|
"loss": 0.6897,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 0.01153564453125,
|
|
"rewards/margins": 0.00640869140625,
|
|
"rewards/rejected": 0.005157470703125,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.017324767198440772,
|
|
"grad_norm": 1.923679325098434,
|
|
"learning_rate": 8.658008658008658e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6889,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.01507568359375,
|
|
"rewards/margins": 0.00933837890625,
|
|
"rewards/rejected": 0.005706787109375,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.018046632498375802,
|
|
"grad_norm": 1.9336109183415302,
|
|
"learning_rate": 9.018759018759018e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -198.0,
|
|
"logps/rejected": -190.0,
|
|
"loss": 0.6883,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.015625,
|
|
"rewards/margins": 0.01055908203125,
|
|
"rewards/rejected": 0.005035400390625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.018768497798310836,
|
|
"grad_norm": 1.7118991587292718,
|
|
"learning_rate": 9.37950937950938e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -1.9921875,
|
|
"logps/chosen": -148.0,
|
|
"logps/rejected": -164.0,
|
|
"loss": 0.6873,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": 0.0172119140625,
|
|
"rewards/margins": 0.012451171875,
|
|
"rewards/rejected": 0.00469970703125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.019490363098245866,
|
|
"grad_norm": 2.0498889647791634,
|
|
"learning_rate": 9.74025974025974e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -150.0,
|
|
"loss": 0.6875,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.00982666015625,
|
|
"rewards/rejected": 0.006195068359375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.0202122283981809,
|
|
"grad_norm": 1.7948780770940682,
|
|
"learning_rate": 1.01010101010101e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -170.0,
|
|
"loss": 0.6865,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": 0.018310546875,
|
|
"rewards/margins": 0.01513671875,
|
|
"rewards/rejected": 0.0032196044921875,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.02093409369811593,
|
|
"grad_norm": 1.7485728753360685,
|
|
"learning_rate": 1.0461760461760462e-07,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -173.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6865,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": 0.0172119140625,
|
|
"rewards/margins": 0.0089111328125,
|
|
"rewards/rejected": 0.0084228515625,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.021655958998050964,
|
|
"grad_norm": 1.7752446010354384,
|
|
"learning_rate": 1.0822510822510822e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6857,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0238037109375,
|
|
"rewards/margins": 0.015625,
|
|
"rewards/rejected": 0.00823974609375,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.022377824297985995,
|
|
"grad_norm": 1.9527079542055057,
|
|
"learning_rate": 1.1183261183261184e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -148.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6845,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.0203857421875,
|
|
"rewards/margins": 0.0184326171875,
|
|
"rewards/rejected": 0.00201416015625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.02309968959792103,
|
|
"grad_norm": 1.798683396295616,
|
|
"learning_rate": 1.1544011544011543e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6844,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 0.02197265625,
|
|
"rewards/margins": 0.0184326171875,
|
|
"rewards/rejected": 0.003570556640625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.02382155489785606,
|
|
"grad_norm": 1.822086025310402,
|
|
"learning_rate": 1.1904761904761903e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.9765625,
|
|
"logps/chosen": -159.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6816,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 0.02490234375,
|
|
"rewards/margins": 0.0225830078125,
|
|
"rewards/rejected": 0.002349853515625,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.024543420197791092,
|
|
"grad_norm": 1.8205804931965275,
|
|
"learning_rate": 1.2265512265512265e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6809,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0240478515625,
|
|
"rewards/margins": 0.02392578125,
|
|
"rewards/rejected": 4.839897155761719e-05,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.025265285497726123,
|
|
"grad_norm": 2.015225469187424,
|
|
"learning_rate": 1.2626262626262626e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -138.0,
|
|
"loss": 0.6787,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": 0.023193359375,
|
|
"rewards/margins": 0.02880859375,
|
|
"rewards/rejected": -0.005706787109375,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.025987150797661156,
|
|
"grad_norm": 1.8960338628946363,
|
|
"learning_rate": 1.2987012987012984e-07,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.679,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": 0.024169921875,
|
|
"rewards/margins": 0.029052734375,
|
|
"rewards/rejected": -0.004791259765625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.02670901609759619,
|
|
"grad_norm": 1.9289106441512014,
|
|
"learning_rate": 1.3347763347763348e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -143.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6755,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 0.0277099609375,
|
|
"rewards/margins": 0.036376953125,
|
|
"rewards/rejected": -0.00848388671875,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.02743088139753122,
|
|
"grad_norm": 2.4318763159683168,
|
|
"learning_rate": 1.370851370851371e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -161.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6738,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": 0.016845703125,
|
|
"rewards/margins": 0.0390625,
|
|
"rewards/rejected": -0.022216796875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.028152746697466254,
|
|
"grad_norm": 1.727518768188907,
|
|
"learning_rate": 1.406926406926407e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6737,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": 0.0238037109375,
|
|
"rewards/margins": 0.04541015625,
|
|
"rewards/rejected": -0.0216064453125,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.028874611997401285,
|
|
"grad_norm": 1.8565593414895172,
|
|
"learning_rate": 1.4430014430014428e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6703,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 0.01324462890625,
|
|
"rewards/margins": 0.0458984375,
|
|
"rewards/rejected": -0.03271484375,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.02959647729733632,
|
|
"grad_norm": 1.9588571879597823,
|
|
"learning_rate": 1.479076479076479e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6681,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 0.006378173828125,
|
|
"rewards/margins": 0.052734375,
|
|
"rewards/rejected": -0.04638671875,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.03031834259727135,
|
|
"grad_norm": 1.93069247030128,
|
|
"learning_rate": 1.5151515151515152e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6667,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -0.0025787353515625,
|
|
"rewards/margins": 0.044189453125,
|
|
"rewards/rejected": -0.046875,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.031040207897206382,
|
|
"grad_norm": 1.819742407006859,
|
|
"learning_rate": 1.5512265512265513e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6625,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.002685546875,
|
|
"rewards/margins": 0.06396484375,
|
|
"rewards/rejected": -0.06640625,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.031762073197141416,
|
|
"grad_norm": 2.3055219231256108,
|
|
"learning_rate": 1.5873015873015872e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6555,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.01123046875,
|
|
"rewards/margins": 0.078125,
|
|
"rewards/rejected": -0.08935546875,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.032483938497076446,
|
|
"grad_norm": 1.9129712763765747,
|
|
"learning_rate": 1.6233766233766232e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6547,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.031494140625,
|
|
"rewards/margins": 0.07666015625,
|
|
"rewards/rejected": -0.10791015625,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.03320580379701148,
|
|
"grad_norm": 1.936887902580248,
|
|
"learning_rate": 1.6594516594516593e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.65,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.0478515625,
|
|
"rewards/margins": 0.08740234375,
|
|
"rewards/rejected": -0.1357421875,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.03392766909694651,
|
|
"grad_norm": 2.1168551608123725,
|
|
"learning_rate": 1.6955266955266957e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -190.0,
|
|
"loss": 0.6399,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.06298828125,
|
|
"rewards/margins": 0.11279296875,
|
|
"rewards/rejected": -0.17578125,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.034649534396881544,
|
|
"grad_norm": 2.4110075564522533,
|
|
"learning_rate": 1.7316017316017315e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -167.0,
|
|
"loss": 0.6345,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.09375,
|
|
"rewards/margins": 0.1376953125,
|
|
"rewards/rejected": -0.2314453125,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.035371399696816574,
|
|
"grad_norm": 2.561479324237141,
|
|
"learning_rate": 1.7676767676767676e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6272,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.1552734375,
|
|
"rewards/margins": 0.1328125,
|
|
"rewards/rejected": -0.287109375,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.036093264996751605,
|
|
"grad_norm": 2.2009284942320004,
|
|
"learning_rate": 1.8037518037518037e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -198.0,
|
|
"loss": 0.6213,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.1826171875,
|
|
"rewards/margins": 0.1806640625,
|
|
"rewards/rejected": -0.36328125,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.036815130296686635,
|
|
"grad_norm": 2.433624086599741,
|
|
"learning_rate": 1.8398268398268395e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -182.0,
|
|
"logps/rejected": -181.0,
|
|
"loss": 0.6137,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.2294921875,
|
|
"rewards/margins": 0.1787109375,
|
|
"rewards/rejected": -0.408203125,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.03753699559662167,
|
|
"grad_norm": 2.3303081056576396,
|
|
"learning_rate": 1.875901875901876e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -183.0,
|
|
"logps/rejected": -198.0,
|
|
"loss": 0.6089,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.28125,
|
|
"rewards/margins": 0.1953125,
|
|
"rewards/rejected": -0.4765625,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.0382588608965567,
|
|
"grad_norm": 7.529255626517495,
|
|
"learning_rate": 1.911976911976912e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -206.0,
|
|
"loss": 0.6063,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.3046875,
|
|
"rewards/margins": 0.236328125,
|
|
"rewards/rejected": -0.5390625,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.03898072619649173,
|
|
"grad_norm": 2.2072494405878578,
|
|
"learning_rate": 1.948051948051948e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -224.0,
|
|
"loss": 0.5956,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.3515625,
|
|
"rewards/margins": 0.283203125,
|
|
"rewards/rejected": -0.6328125,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.03970259149642677,
|
|
"grad_norm": 2.310476086986689,
|
|
"learning_rate": 1.984126984126984e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -220.0,
|
|
"logps/rejected": -209.0,
|
|
"loss": 0.5867,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.416015625,
|
|
"rewards/margins": 0.236328125,
|
|
"rewards/rejected": -0.65234375,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.0404244567963618,
|
|
"grad_norm": 2.6001658124806286,
|
|
"learning_rate": 2.02020202020202e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -208.0,
|
|
"logps/rejected": -239.0,
|
|
"loss": 0.5754,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.447265625,
|
|
"rewards/margins": 0.31640625,
|
|
"rewards/rejected": -0.76171875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.04114632209629683,
|
|
"grad_norm": 2.635075269431316,
|
|
"learning_rate": 2.0562770562770563e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -217.0,
|
|
"logps/rejected": -227.0,
|
|
"loss": 0.5719,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.494140625,
|
|
"rewards/margins": 0.33984375,
|
|
"rewards/rejected": -0.83203125,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.04186818739623186,
|
|
"grad_norm": 2.5380650062224066,
|
|
"learning_rate": 2.0923520923520924e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -216.0,
|
|
"logps/rejected": -237.0,
|
|
"loss": 0.5794,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.50390625,
|
|
"rewards/margins": 0.349609375,
|
|
"rewards/rejected": -0.8515625,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.0425900526961669,
|
|
"grad_norm": 2.808866971835476,
|
|
"learning_rate": 2.1284271284271282e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -216.0,
|
|
"logps/rejected": -264.0,
|
|
"loss": 0.5557,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.5390625,
|
|
"rewards/margins": 0.50390625,
|
|
"rewards/rejected": -1.046875,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.04331191799610193,
|
|
"grad_norm": 3.660790591856929,
|
|
"learning_rate": 2.1645021645021643e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -215.0,
|
|
"logps/rejected": -226.0,
|
|
"loss": 0.5374,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.60546875,
|
|
"rewards/margins": 0.3203125,
|
|
"rewards/rejected": -0.92578125,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.04403378329603696,
|
|
"grad_norm": 3.4012692341173842,
|
|
"learning_rate": 2.2005772005772004e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -243.0,
|
|
"logps/rejected": -288.0,
|
|
"loss": 0.5302,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.75,
|
|
"rewards/margins": 0.51953125,
|
|
"rewards/rejected": -1.2734375,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.04475564859597199,
|
|
"grad_norm": 3.1213723367337276,
|
|
"learning_rate": 2.2366522366522368e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -252.0,
|
|
"logps/rejected": -290.0,
|
|
"loss": 0.5349,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.82421875,
|
|
"rewards/margins": 0.6171875,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.045477513895907026,
|
|
"grad_norm": 3.4139219834989825,
|
|
"learning_rate": 2.2727272727272726e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -241.0,
|
|
"logps/rejected": -290.0,
|
|
"loss": 0.5313,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.76953125,
|
|
"rewards/margins": 0.625,
|
|
"rewards/rejected": -1.390625,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.04619937919584206,
|
|
"grad_norm": 3.545601457816913,
|
|
"learning_rate": 2.3088023088023087e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -254.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5163,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.796875,
|
|
"rewards/margins": 0.671875,
|
|
"rewards/rejected": -1.46875,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.04692124449577709,
|
|
"grad_norm": 6.320535114176617,
|
|
"learning_rate": 2.3448773448773448e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -258.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.5168,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -0.92578125,
|
|
"rewards/margins": 0.625,
|
|
"rewards/rejected": -1.5546875,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.04764310979571212,
|
|
"grad_norm": 4.40606882426551,
|
|
"learning_rate": 2.3809523809523806e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -258.0,
|
|
"logps/rejected": -300.0,
|
|
"loss": 0.506,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.9375,
|
|
"rewards/margins": 0.6171875,
|
|
"rewards/rejected": -1.5546875,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.048364975095647154,
|
|
"grad_norm": 7.228420353041346,
|
|
"learning_rate": 2.4170274170274167e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -230.0,
|
|
"logps/rejected": -308.0,
|
|
"loss": 0.5231,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.8359375,
|
|
"rewards/margins": 0.7421875,
|
|
"rewards/rejected": -1.578125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.049086840395582185,
|
|
"grad_norm": 4.810449590822843,
|
|
"learning_rate": 2.453102453102453e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -246.0,
|
|
"logps/rejected": -280.0,
|
|
"loss": 0.5107,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.88671875,
|
|
"rewards/margins": 0.498046875,
|
|
"rewards/rejected": -1.3828125,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.049808705695517215,
|
|
"grad_norm": 6.916956622472092,
|
|
"learning_rate": 2.4891774891774894e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -270.0,
|
|
"logps/rejected": -322.0,
|
|
"loss": 0.5105,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.9765625,
|
|
"rewards/margins": 0.6484375,
|
|
"rewards/rejected": -1.625,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.050530570995452245,
|
|
"grad_norm": 5.072218871042774,
|
|
"learning_rate": 2.525252525252525e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -256.0,
|
|
"logps/rejected": -326.0,
|
|
"loss": 0.5065,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.92578125,
|
|
"rewards/margins": 0.76953125,
|
|
"rewards/rejected": -1.6953125,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.05125243629538728,
|
|
"grad_norm": 8.875662849487487,
|
|
"learning_rate": 2.5613275613275616e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -254.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5112,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.94921875,
|
|
"rewards/margins": 0.61328125,
|
|
"rewards/rejected": -1.5625,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.05197430159532231,
|
|
"grad_norm": 5.595849818392523,
|
|
"learning_rate": 2.597402597402597e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -264.0,
|
|
"logps/rejected": -340.0,
|
|
"loss": 0.5107,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.0078125,
|
|
"rewards/margins": 0.8125,
|
|
"rewards/rejected": -1.8203125,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.05269616689525734,
|
|
"grad_norm": 5.404147285645744,
|
|
"learning_rate": 2.633477633477633e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -368.0,
|
|
"loss": 0.5035,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.984375,
|
|
"rewards/margins": 0.94140625,
|
|
"rewards/rejected": -1.9296875,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.05341803219519238,
|
|
"grad_norm": 8.76041109611346,
|
|
"learning_rate": 2.6695526695526696e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -342.0,
|
|
"loss": 0.4887,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -1.125,
|
|
"rewards/margins": 0.75390625,
|
|
"rewards/rejected": -1.8828125,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.05413989749512741,
|
|
"grad_norm": 7.025589443972428,
|
|
"learning_rate": 2.7056277056277054e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -262.0,
|
|
"logps/rejected": -348.0,
|
|
"loss": 0.4823,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.0,
|
|
"rewards/margins": 0.94921875,
|
|
"rewards/rejected": -1.953125,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.05486176279506244,
|
|
"grad_norm": 11.975302977639574,
|
|
"learning_rate": 2.741702741702742e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -322.0,
|
|
"loss": 0.4766,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.140625,
|
|
"rewards/margins": 0.6953125,
|
|
"rewards/rejected": -1.8359375,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.05558362809499747,
|
|
"grad_norm": 6.5118457359444,
|
|
"learning_rate": 2.7777777777777776e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -346.0,
|
|
"loss": 0.4846,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.1875,
|
|
"rewards/margins": 0.80859375,
|
|
"rewards/rejected": -1.9921875,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.05630549339493251,
|
|
"grad_norm": 6.106781577833728,
|
|
"learning_rate": 2.813852813852814e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -260.0,
|
|
"logps/rejected": -356.0,
|
|
"loss": 0.4665,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.046875,
|
|
"rewards/margins": 0.83203125,
|
|
"rewards/rejected": -1.875,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.05702735869486754,
|
|
"grad_norm": 10.75495308386994,
|
|
"learning_rate": 2.8499278499278503e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -284.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4695,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.2109375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.390625,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.05774922399480257,
|
|
"grad_norm": 5.788421207967755,
|
|
"learning_rate": 2.8860028860028856e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.4767,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.125,
|
|
"rewards/margins": 0.84765625,
|
|
"rewards/rejected": -1.9765625,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.0584710892947376,
|
|
"grad_norm": 8.78614857490401,
|
|
"learning_rate": 2.922077922077922e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -262.0,
|
|
"logps/rejected": -362.0,
|
|
"loss": 0.4722,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -2.125,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.05919295459467264,
|
|
"grad_norm": 7.3161771388355294,
|
|
"learning_rate": 2.958152958152958e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.4996,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 0.93359375,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.05991481989460767,
|
|
"grad_norm": 8.266553993044619,
|
|
"learning_rate": 2.994227994227994e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.4545,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 1.015625,
|
|
"rewards/rejected": -2.171875,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.0606366851945427,
|
|
"grad_norm": 8.218438777679667,
|
|
"learning_rate": 3.0303030303030305e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -382.0,
|
|
"loss": 0.4639,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -2.28125,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.06135855049447773,
|
|
"grad_norm": 8.61378341353772,
|
|
"learning_rate": 3.0663780663780663e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -284.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4536,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.2578125,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.062080415794412765,
|
|
"grad_norm": 12.398406848670698,
|
|
"learning_rate": 3.1024531024531027e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.476,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.2578125,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -2.265625,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.0628022810943478,
|
|
"grad_norm": 12.244121744087602,
|
|
"learning_rate": 3.138528138528138e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -388.0,
|
|
"loss": 0.4623,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.06352414639428283,
|
|
"grad_norm": 10.812869810884447,
|
|
"learning_rate": 3.1746031746031743e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -366.0,
|
|
"loss": 0.4498,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.99609375,
|
|
"rewards/rejected": -2.234375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.06424601169421786,
|
|
"grad_norm": 11.229212598361977,
|
|
"learning_rate": 3.2106782106782107e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4373,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.3828125,
|
|
"rewards/margins": 1.0859375,
|
|
"rewards/rejected": -2.46875,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.06496787699415289,
|
|
"grad_norm": 7.414765021326351,
|
|
"learning_rate": 3.2467532467532465e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4253,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -2.484375,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.06568974229408793,
|
|
"grad_norm": 6.656174306105041,
|
|
"learning_rate": 3.282828282828283e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -440.0,
|
|
"loss": 0.4366,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.06641160759402295,
|
|
"grad_norm": 6.986955189515223,
|
|
"learning_rate": 3.3189033189033187e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -422.0,
|
|
"loss": 0.4335,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -2.75,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.06713347289395799,
|
|
"grad_norm": 8.704691417598838,
|
|
"learning_rate": 3.354978354978355e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.0703125,
|
|
"rewards/rejected": -2.640625,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.06785533819389301,
|
|
"grad_norm": 19.52535270358171,
|
|
"learning_rate": 3.3910533910533914e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.434,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.6015625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.06857720349382805,
|
|
"grad_norm": 11.339737953685198,
|
|
"learning_rate": 3.4271284271284267e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -398.0,
|
|
"loss": 0.4334,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.5625,
|
|
"rewards/margins": 1.03125,
|
|
"rewards/rejected": -2.59375,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.06929906879376309,
|
|
"grad_norm": 10.31037341226185,
|
|
"learning_rate": 3.463203463203463e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -3.046875,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.07002093409369811,
|
|
"grad_norm": 11.130584733664083,
|
|
"learning_rate": 3.499278499278499e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -424.0,
|
|
"loss": 0.4078,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.4296875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -2.796875,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.07074279939363315,
|
|
"grad_norm": 9.06083640139615,
|
|
"learning_rate": 3.535353535353535e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -442.0,
|
|
"loss": 0.4178,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -2.890625,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.07146466469356819,
|
|
"grad_norm": 9.541422651232944,
|
|
"learning_rate": 3.5714285714285716e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.3953,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -3.078125,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.07218652999350321,
|
|
"grad_norm": 9.110551563711448,
|
|
"learning_rate": 3.6075036075036074e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -464.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -2.984375,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.07290839529343825,
|
|
"grad_norm": 15.071837561832831,
|
|
"learning_rate": 3.643578643578644e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.4045,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5625,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -2.921875,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.07363026059337327,
|
|
"grad_norm": 13.920977378141336,
|
|
"learning_rate": 3.679653679653679e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.431,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.8125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -3.28125,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.07435212589330831,
|
|
"grad_norm": 11.085289038392293,
|
|
"learning_rate": 3.7157287157287154e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.3934,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.07507399119324334,
|
|
"grad_norm": 10.088971897286424,
|
|
"learning_rate": 3.751803751803752e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4073,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.07579585649317837,
|
|
"grad_norm": 43.24163673888106,
|
|
"learning_rate": 3.7878787878787876e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -496.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.890625,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -3.140625,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.0765177217931134,
|
|
"grad_norm": 8.537892452168746,
|
|
"learning_rate": 3.823953823953824e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -480.0,
|
|
"loss": 0.3947,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.765625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.07723958709304844,
|
|
"grad_norm": 9.485771716655114,
|
|
"learning_rate": 3.86002886002886e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.3837,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -3.09375,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.07796145239298347,
|
|
"grad_norm": 9.46365785011617,
|
|
"learning_rate": 3.896103896103896e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.15625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.9375,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.0786833176929185,
|
|
"grad_norm": 13.461346342787172,
|
|
"learning_rate": 3.9321789321789325e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -360.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -3.765625,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.07940518299285354,
|
|
"grad_norm": 12.44634151581957,
|
|
"learning_rate": 3.968253968253968e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -508.0,
|
|
"loss": 0.3727,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -2.015625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -3.515625,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.08012704829278856,
|
|
"grad_norm": 11.687822831294243,
|
|
"learning_rate": 4.004329004329004e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -568.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.0625,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.0808489135927236,
|
|
"grad_norm": 16.193115895860167,
|
|
"learning_rate": 4.04040404040404e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -532.0,
|
|
"loss": 0.3726,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.9921875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.08157077889265862,
|
|
"grad_norm": 10.454280347519187,
|
|
"learning_rate": 4.0764790764790763e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -350.0,
|
|
"logps/rejected": -496.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.78125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.08229264419259366,
|
|
"grad_norm": 10.033847939954763,
|
|
"learning_rate": 4.1125541125541127e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.355,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.7890625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -3.1875,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.0830145094925287,
|
|
"grad_norm": 13.538065077526934,
|
|
"learning_rate": 4.1486291486291485e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -398.0,
|
|
"logps/rejected": -588.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.1875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.08373637479246372,
|
|
"grad_norm": 12.06165274825125,
|
|
"learning_rate": 4.184704184704185e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -500.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.6796875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.08445824009239876,
|
|
"grad_norm": 14.245004620424362,
|
|
"learning_rate": 4.22077922077922e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -376.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3599,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.21875,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.0851801053923338,
|
|
"grad_norm": 12.93467746542696,
|
|
"learning_rate": 4.2568542568542565e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -330.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.3635,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.75,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -3.34375,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.08590197069226882,
|
|
"grad_norm": 11.202101570401776,
|
|
"learning_rate": 4.292929292929293e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.3583,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -3.953125,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.08662383599220386,
|
|
"grad_norm": 10.771994466129188,
|
|
"learning_rate": 4.3290043290043287e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -358.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.3465,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.8828125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.08734570129213888,
|
|
"grad_norm": 14.841268368598595,
|
|
"learning_rate": 4.365079365079365e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3477,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.08806756659207392,
|
|
"grad_norm": 9.504108427187644,
|
|
"learning_rate": 4.401154401154401e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -394.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3755,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.25,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.08878943189200895,
|
|
"grad_norm": 12.115514229718103,
|
|
"learning_rate": 4.437229437229437e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -304.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3704,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -1.5390625,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -3.71875,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.08951129719194398,
|
|
"grad_norm": 10.317228267259349,
|
|
"learning_rate": 4.4733044733044736e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3559,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.09023316249187902,
|
|
"grad_norm": 11.197522859178363,
|
|
"learning_rate": 4.509379509379509e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3219,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.09095502779181405,
|
|
"grad_norm": 13.497331724381818,
|
|
"learning_rate": 4.545454545454545e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3388,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.546875,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.09167689309174908,
|
|
"grad_norm": 11.433124284340494,
|
|
"learning_rate": 4.581529581529581e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.3458,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -4.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.09239875839168411,
|
|
"grad_norm": 12.324384516756012,
|
|
"learning_rate": 4.6176046176046174e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -410.0,
|
|
"logps/rejected": -580.0,
|
|
"loss": 0.3426,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.09312062369161915,
|
|
"grad_norm": 10.730987022263216,
|
|
"learning_rate": 4.6536796536796537e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3593,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -4.625,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.09384248899155417,
|
|
"grad_norm": 10.132938191746307,
|
|
"learning_rate": 4.6897546897546896e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -408.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3632,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.359375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.09456435429148921,
|
|
"grad_norm": 11.988621181927288,
|
|
"learning_rate": 4.725829725829726e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.3609,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.921875,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.09528621959142423,
|
|
"grad_norm": 8.802084815882347,
|
|
"learning_rate": 4.761904761904761e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3197,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.9375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.09600808489135927,
|
|
"grad_norm": 13.978035740824382,
|
|
"learning_rate": 4.797979797979798e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.355,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.09672995019129431,
|
|
"grad_norm": 9.329953252240916,
|
|
"learning_rate": 4.834054834054833e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3382,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.09745181549122933,
|
|
"grad_norm": 14.354032725201476,
|
|
"learning_rate": 4.87012987012987e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3375,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.53125,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.09817368079116437,
|
|
"grad_norm": 11.184165035806638,
|
|
"learning_rate": 4.906204906204906e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -640.0,
|
|
"loss": 0.3528,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.0988955460910994,
|
|
"grad_norm": 9.32730305340813,
|
|
"learning_rate": 4.942279942279942e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.3343,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.09961741139103443,
|
|
"grad_norm": 17.291834755334516,
|
|
"learning_rate": 4.978354978354979e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3246,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.10033927669096947,
|
|
"grad_norm": 10.743034154219572,
|
|
"learning_rate": 4.999998729993963e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3335,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.10106114199090449,
|
|
"grad_norm": 17.190340388139088,
|
|
"learning_rate": 4.999984442440868e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3204,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.10178300729083953,
|
|
"grad_norm": 9.641498367561857,
|
|
"learning_rate": 4.99995427991816e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3379,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.10250487259077457,
|
|
"grad_norm": 9.768069678095703,
|
|
"learning_rate": 4.999908242617371e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.34,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.10322673789070959,
|
|
"grad_norm": 13.37422488078675,
|
|
"learning_rate": 4.99984633083084e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.331,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.10394860319064463,
|
|
"grad_norm": 11.253293027609532,
|
|
"learning_rate": 4.99976854495171e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.3218,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.453125,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.10467046849057966,
|
|
"grad_norm": 10.82613372416563,
|
|
"learning_rate": 4.999674885473922e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3213,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.921875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -4.65625,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.10539233379051469,
|
|
"grad_norm": 15.794446739298891,
|
|
"learning_rate": 4.999565352992216e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.3289,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.10611419909044972,
|
|
"grad_norm": 10.361664841175285,
|
|
"learning_rate": 4.999439948202127e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.3261,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.10683606439038476,
|
|
"grad_norm": 10.555951221100015,
|
|
"learning_rate": 4.999298671899977e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3399,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.10755792969031978,
|
|
"grad_norm": 9.802204192641314,
|
|
"learning_rate": 4.999141524982877e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.309,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.10827979499025482,
|
|
"grad_norm": 12.72303480502336,
|
|
"learning_rate": 4.998968508448714e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -632.0,
|
|
"loss": 0.3276,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -4.78125,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.10900166029018984,
|
|
"grad_norm": 14.570251314538908,
|
|
"learning_rate": 4.998779623396146e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.2995,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.10972352559012488,
|
|
"grad_norm": 11.169758259343533,
|
|
"learning_rate": 4.9985748710246e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3344,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.11044539089005992,
|
|
"grad_norm": 10.504554948014457,
|
|
"learning_rate": 4.998354252634257e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -486.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.2853,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.11116725618999494,
|
|
"grad_norm": 15.073212833972551,
|
|
"learning_rate": 4.998117769626051e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.309,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.11188912148992998,
|
|
"grad_norm": 10.298569511142073,
|
|
"learning_rate": 4.997865423501657e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -380.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3261,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.11261098678986502,
|
|
"grad_norm": 11.022262328486809,
|
|
"learning_rate": 4.997597215863477e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2878,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.11333285208980004,
|
|
"grad_norm": 12.425423187064853,
|
|
"learning_rate": 4.99731314841464e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.3008,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.11405471738973508,
|
|
"grad_norm": 10.996930809431353,
|
|
"learning_rate": 4.997013222958978e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.3105,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.1147765826896701,
|
|
"grad_norm": 11.11449736666811,
|
|
"learning_rate": 4.99669744140103e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.3043,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.11549844798960514,
|
|
"grad_norm": 10.155961204890554,
|
|
"learning_rate": 4.996365805746015e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -474.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2908,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.11622031328954018,
|
|
"grad_norm": 9.079844890608808,
|
|
"learning_rate": 4.996018318099829e-07,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2977,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.1169421785894752,
|
|
"grad_norm": 13.337729397169802,
|
|
"learning_rate": 4.995654980669028e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2937,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.03125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.11766404388941024,
|
|
"grad_norm": 9.987891055540013,
|
|
"learning_rate": 4.995275795760816e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.2991,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.11838590918934527,
|
|
"grad_norm": 12.804317711855225,
|
|
"learning_rate": 4.994880765783026e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2883,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.1191077744892803,
|
|
"grad_norm": 11.589955875632223,
|
|
"learning_rate": 4.99446989324411e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2847,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.11982963978921533,
|
|
"grad_norm": 14.992092750756044,
|
|
"learning_rate": 4.99404318075312e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2926,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.12055150508915037,
|
|
"grad_norm": 14.63652475825442,
|
|
"learning_rate": 4.993600631019689e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3115,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.1212733703890854,
|
|
"grad_norm": 10.01896442134856,
|
|
"learning_rate": 4.993142246854024e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2917,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.12199523568902043,
|
|
"grad_norm": 10.480137264657502,
|
|
"learning_rate": 4.992668031166876e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.3114,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.12271710098895545,
|
|
"grad_norm": 9.543166172183264,
|
|
"learning_rate": 4.992177986969526e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.3235,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.12343896628889049,
|
|
"grad_norm": 9.39264085167921,
|
|
"learning_rate": 4.991672117373769e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2971,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.12416083158882553,
|
|
"grad_norm": 9.35509206624716,
|
|
"learning_rate": 4.991150425591891e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.2908,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.12488269688876055,
|
|
"grad_norm": 9.262201280713281,
|
|
"learning_rate": 4.99061291493665e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2925,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.1256045621886956,
|
|
"grad_norm": 10.008148122591312,
|
|
"learning_rate": 4.99005958882125e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.3133,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.12632642748863063,
|
|
"grad_norm": 8.554136981806407,
|
|
"learning_rate": 4.989490450759331e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2649,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.12704829278856566,
|
|
"grad_norm": 8.929491173417231,
|
|
"learning_rate": 4.988905504364933e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2715,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.12777015808850067,
|
|
"grad_norm": 19.85583075822656,
|
|
"learning_rate": 4.988304753352482e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.046875,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.1284920233884357,
|
|
"grad_norm": 11.573857511888404,
|
|
"learning_rate": 4.987688201536764e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3207,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.12921388868837075,
|
|
"grad_norm": 10.767926076980798,
|
|
"learning_rate": 4.987055852832899e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2768,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.296875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.12993575398830579,
|
|
"grad_norm": 9.480524665881354,
|
|
"learning_rate": 4.986407711256319e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2996,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.13065761928824082,
|
|
"grad_norm": 11.059595953397832,
|
|
"learning_rate": 4.98574378092274e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2706,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.13137948458817586,
|
|
"grad_norm": 12.559652259820835,
|
|
"learning_rate": 4.985064066048139e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.3016,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.13210134988811087,
|
|
"grad_norm": 16.780300866871737,
|
|
"learning_rate": 4.984368570948724e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.2943,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.1328232151880459,
|
|
"grad_norm": 10.65248837076819,
|
|
"learning_rate": 4.983657300040907e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.2699,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.13354508048798094,
|
|
"grad_norm": 10.948003648298677,
|
|
"learning_rate": 4.982930257841278e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2941,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.13426694578791598,
|
|
"grad_norm": 12.047287233162004,
|
|
"learning_rate": 4.982187448966575e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2898,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.640625,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.13498881108785102,
|
|
"grad_norm": 14.9941388186048,
|
|
"learning_rate": 4.981428878133656e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2811,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.13571067638778603,
|
|
"grad_norm": 9.951835424657778,
|
|
"learning_rate": 4.980654550159463e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.2732,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.13643254168772107,
|
|
"grad_norm": 8.861361984310934,
|
|
"learning_rate": 4.979864469961004e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2762,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.1371544069876561,
|
|
"grad_norm": 9.692735634395657,
|
|
"learning_rate": 4.979058642555307e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2931,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.13787627228759114,
|
|
"grad_norm": 9.565594633496087,
|
|
"learning_rate": 4.978237073059399e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2898,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.13859813758752618,
|
|
"grad_norm": 11.007909649107406,
|
|
"learning_rate": 4.977399766690269e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2902,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.1393200028874612,
|
|
"grad_norm": 9.107456714393756,
|
|
"learning_rate": 4.976546728764836e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.284,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.14004186818739622,
|
|
"grad_norm": 8.927298343498093,
|
|
"learning_rate": 4.975677964699912e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2524,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.14076373348733126,
|
|
"grad_norm": 10.144209347121677,
|
|
"learning_rate": 4.974793480012174e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2621,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.1414855987872663,
|
|
"grad_norm": 7.533895015502329,
|
|
"learning_rate": 4.973893280318125e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2844,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.14220746408720134,
|
|
"grad_norm": 13.017212573238576,
|
|
"learning_rate": 4.972977371334056e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2873,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.14292932938713637,
|
|
"grad_norm": 8.958119059657708,
|
|
"learning_rate": 4.972045758876014e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2477,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.14365119468707138,
|
|
"grad_norm": 10.493203877125167,
|
|
"learning_rate": 4.971098448859766e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.2933,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.14437305998700642,
|
|
"grad_norm": 9.973862051012356,
|
|
"learning_rate": 4.970135447300752e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2595,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.14509492528694146,
|
|
"grad_norm": 13.28264552294119,
|
|
"learning_rate": 4.969156760314064e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2904,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.640625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.1458167905868765,
|
|
"grad_norm": 8.872169286182675,
|
|
"learning_rate": 4.968162394114387e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2523,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.14653865588681153,
|
|
"grad_norm": 12.757195149734072,
|
|
"learning_rate": 4.967152355015974e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2783,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.14726052118674654,
|
|
"grad_norm": 12.796953074989913,
|
|
"learning_rate": 4.966126649432603e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2723,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.53125,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.14798238648668158,
|
|
"grad_norm": 19.108616982264344,
|
|
"learning_rate": 4.96508528387753e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2763,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.14870425178661661,
|
|
"grad_norm": 9.380783201026548,
|
|
"learning_rate": 4.964028264963456e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.14942611708655165,
|
|
"grad_norm": 10.81595001993398,
|
|
"learning_rate": 4.962955599402481e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.266,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -6.25,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.1501479823864867,
|
|
"grad_norm": 11.456291625001526,
|
|
"learning_rate": 4.961867294006059e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.3064,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.15086984768642173,
|
|
"grad_norm": 7.960182686516041,
|
|
"learning_rate": 4.96076335568496e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2467,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.15159171298635674,
|
|
"grad_norm": 10.25400623338689,
|
|
"learning_rate": 4.959643791449222e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2977,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.375,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.15231357828629177,
|
|
"grad_norm": 10.09340211565806,
|
|
"learning_rate": 4.958508608408109e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2829,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.1530354435862268,
|
|
"grad_norm": 8.537784621679538,
|
|
"learning_rate": 4.957357813770064e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.15375730888616185,
|
|
"grad_norm": 8.985222338453365,
|
|
"learning_rate": 4.956191414842665e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2629,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.15447917418609688,
|
|
"grad_norm": 10.993021927824719,
|
|
"learning_rate": 4.955009419032575e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2681,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.1552010394860319,
|
|
"grad_norm": 11.792550816411442,
|
|
"learning_rate": 4.953811833845503e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.265,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.15592290478596693,
|
|
"grad_norm": 13.163433586155344,
|
|
"learning_rate": 4.952598666886145e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2516,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.15664477008590197,
|
|
"grad_norm": 15.36607001210862,
|
|
"learning_rate": 4.951369925858147e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.2598,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.157366635385837,
|
|
"grad_norm": 8.701540096554936,
|
|
"learning_rate": 4.950125618564046e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.251,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.625,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.15808850068577204,
|
|
"grad_norm": 10.616304811087307,
|
|
"learning_rate": 4.948865752905228e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2533,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.15881036598570708,
|
|
"grad_norm": 10.818629515177625,
|
|
"learning_rate": 4.947590336881874e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2593,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.1595322312856421,
|
|
"grad_norm": 10.595421594796772,
|
|
"learning_rate": 4.946299378592912e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2795,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.578125,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.16025409658557713,
|
|
"grad_norm": 13.24386115549115,
|
|
"learning_rate": 4.944992886235961e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2521,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.16097596188551216,
|
|
"grad_norm": 8.619302388111215,
|
|
"learning_rate": 4.943670868107284e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.27,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.1616978271854472,
|
|
"grad_norm": 8.272453046055942,
|
|
"learning_rate": 4.942333332601731e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.16241969248538224,
|
|
"grad_norm": 6.171166558818759,
|
|
"learning_rate": 4.940980288212691e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2574,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.16314155778531725,
|
|
"grad_norm": 9.42178572046336,
|
|
"learning_rate": 4.939611743532031e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2385,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.16386342308525229,
|
|
"grad_norm": 8.438515261081413,
|
|
"learning_rate": 4.93822770725005e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2737,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.16458528838518732,
|
|
"grad_norm": 11.043392051990788,
|
|
"learning_rate": 4.936828188155413e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.625,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.16530715368512236,
|
|
"grad_norm": 10.831257780639715,
|
|
"learning_rate": 4.935413195135106e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.1660290189850574,
|
|
"grad_norm": 9.747157511340994,
|
|
"learning_rate": 4.933982737174374e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2701,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.1667508842849924,
|
|
"grad_norm": 9.870871070277088,
|
|
"learning_rate": 4.932536823356664e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2543,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.16747274958492744,
|
|
"grad_norm": 8.8575895636107,
|
|
"learning_rate": 4.931075462863567e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2594,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.16819461488486248,
|
|
"grad_norm": 7.816395810688846,
|
|
"learning_rate": 4.929598664974762e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.16891648018479752,
|
|
"grad_norm": 8.344064880799563,
|
|
"learning_rate": 4.928106439067958e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2669,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.16963834548473256,
|
|
"grad_norm": 10.994252200422187,
|
|
"learning_rate": 4.926598794618829e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.278,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.1703602107846676,
|
|
"grad_norm": 15.14815293816792,
|
|
"learning_rate": 4.92507574120096e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2777,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.1710820760846026,
|
|
"grad_norm": 7.821370479620557,
|
|
"learning_rate": 4.923537288485779e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2638,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.17180394138453764,
|
|
"grad_norm": 9.211826498394792,
|
|
"learning_rate": 4.921983446242506e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2493,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.17252580668447268,
|
|
"grad_norm": 8.298256232187494,
|
|
"learning_rate": 4.92041422433808e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2348,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.17324767198440771,
|
|
"grad_norm": 10.983625039262604,
|
|
"learning_rate": 4.918829632737103e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2633,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.17396953728434275,
|
|
"grad_norm": 12.61039915689691,
|
|
"learning_rate": 4.917229681501774e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.264,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.17469140258427776,
|
|
"grad_norm": 11.643682996836732,
|
|
"learning_rate": 4.91561438079183e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2668,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.1754132678842128,
|
|
"grad_norm": 13.088952482660957,
|
|
"learning_rate": 4.913983740864473e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2359,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.17613513318414784,
|
|
"grad_norm": 10.188119267424455,
|
|
"learning_rate": 4.91233777207431e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2748,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.17685699848408287,
|
|
"grad_norm": 8.609826083562075,
|
|
"learning_rate": 4.910676484873292e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2658,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.1775788637840179,
|
|
"grad_norm": 10.48253643988837,
|
|
"learning_rate": 4.908999889810633e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2331,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.17830072908395295,
|
|
"grad_norm": 17.344352556263694,
|
|
"learning_rate": 4.907307997532761e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2354,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.17902259438388796,
|
|
"grad_norm": 9.982854645114383,
|
|
"learning_rate": 4.905600818783237e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3273,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.179744459683823,
|
|
"grad_norm": 10.68094022282627,
|
|
"learning_rate": 4.903878364402691e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2468,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.18046632498375803,
|
|
"grad_norm": 8.296383042780498,
|
|
"learning_rate": 4.902140645328759e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2599,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.18118819028369307,
|
|
"grad_norm": 9.103787824302069,
|
|
"learning_rate": 4.900387672596003e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2318,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.1819100555836281,
|
|
"grad_norm": 16.67988360600736,
|
|
"learning_rate": 4.898619457335848e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.248,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.18263192088356311,
|
|
"grad_norm": 10.469848545031832,
|
|
"learning_rate": 4.896836010776509e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2595,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.18335378618349815,
|
|
"grad_norm": 9.661637065408652,
|
|
"learning_rate": 4.895037344242921e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2585,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.1840756514834332,
|
|
"grad_norm": 8.853616581850865,
|
|
"learning_rate": 4.893223469156664e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.249,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.18479751678336823,
|
|
"grad_norm": 14.23122096213618,
|
|
"learning_rate": 4.891394397035896e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2458,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.18551938208330326,
|
|
"grad_norm": 7.165047283079565,
|
|
"learning_rate": 4.889550139495275e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2193,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.1862412473832383,
|
|
"grad_norm": 9.892022723104382,
|
|
"learning_rate": 4.887690708245887e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2276,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.765625,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.1869631126831733,
|
|
"grad_norm": 9.543867068973059,
|
|
"learning_rate": 4.885816115095171e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2391,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.18768497798310835,
|
|
"grad_norm": 10.889738728754864,
|
|
"learning_rate": 4.883926371946843e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2702,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.18840684328304338,
|
|
"grad_norm": 6.728294158129367,
|
|
"learning_rate": 4.882021490800826e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.18912870858297842,
|
|
"grad_norm": 9.374612134195704,
|
|
"learning_rate": 4.880101483753167e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2198,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.18985057388291346,
|
|
"grad_norm": 8.403906697614635,
|
|
"learning_rate": 4.878166362995963e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2728,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.19057243918284847,
|
|
"grad_norm": 9.739452944604345,
|
|
"learning_rate": 4.876216140817285e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2399,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.1912943044827835,
|
|
"grad_norm": 9.412524327526398,
|
|
"learning_rate": 4.874250829601094e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.19201616978271854,
|
|
"grad_norm": 9.67143825795057,
|
|
"learning_rate": 4.872270441827174e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2358,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.19273803508265358,
|
|
"grad_norm": 10.225198499138108,
|
|
"learning_rate": 4.870274990071038e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.19345990038258862,
|
|
"grad_norm": 8.93422692887298,
|
|
"learning_rate": 4.868264487003862e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2518,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.19418176568252365,
|
|
"grad_norm": 8.108160852039612,
|
|
"learning_rate": 4.866238945392393e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2465,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.19490363098245866,
|
|
"grad_norm": 8.305890952481619,
|
|
"learning_rate": 4.864198378098877e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.237,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.1956254962823937,
|
|
"grad_norm": 12.954558843929119,
|
|
"learning_rate": 4.862142798080973e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2245,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.19634736158232874,
|
|
"grad_norm": 6.775612276163364,
|
|
"learning_rate": 4.860072218391669e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2153,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.19706922688226378,
|
|
"grad_norm": 8.770593576643066,
|
|
"learning_rate": 4.857986652179203e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2534,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.1977910921821988,
|
|
"grad_norm": 8.729235176349604,
|
|
"learning_rate": 4.855886112686977e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.19851295748213382,
|
|
"grad_norm": 10.466383153721448,
|
|
"learning_rate": 4.853770613253476e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.19923482278206886,
|
|
"grad_norm": 10.738593768361735,
|
|
"learning_rate": 4.851640167312178e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.224,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.1999566880820039,
|
|
"grad_norm": 9.416300695572696,
|
|
"learning_rate": 4.849494788391473e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2284,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.20067855338193893,
|
|
"grad_norm": 9.528669823636646,
|
|
"learning_rate": 4.847334490114576e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2293,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.20140041868187397,
|
|
"grad_norm": 9.797059355953671,
|
|
"learning_rate": 4.84515928619944e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2094,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.20212228398180898,
|
|
"grad_norm": 9.406533092835442,
|
|
"learning_rate": 4.84296919045867e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2334,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.20284414928174402,
|
|
"grad_norm": 9.254549987899654,
|
|
"learning_rate": 4.840764216799433e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2337,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.20356601458167906,
|
|
"grad_norm": 9.14538857003236,
|
|
"learning_rate": 4.838544379223373e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2619,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.2042878798816141,
|
|
"grad_norm": 9.535277544276601,
|
|
"learning_rate": 4.836309691826518e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.238,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.20500974518154913,
|
|
"grad_norm": 8.327834581371992,
|
|
"learning_rate": 4.834060168799195e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2305,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.20573161048148417,
|
|
"grad_norm": 7.7347670431429405,
|
|
"learning_rate": 4.831795824425938e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2542,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.20645347578141918,
|
|
"grad_norm": 8.15521635648461,
|
|
"learning_rate": 4.829516673085394e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.20717534108135421,
|
|
"grad_norm": 8.022215432273788,
|
|
"learning_rate": 4.827222729250236e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2461,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.20789720638128925,
|
|
"grad_norm": 7.199943747449723,
|
|
"learning_rate": 4.824914007487072e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2155,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.2086190716812243,
|
|
"grad_norm": 12.583865202253415,
|
|
"learning_rate": 4.822590522456347e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2309,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.20934093698115933,
|
|
"grad_norm": 10.061884816438084,
|
|
"learning_rate": 4.820252288912254e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.21006280228109434,
|
|
"grad_norm": 8.24310045419468,
|
|
"learning_rate": 4.817899321702642e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2341,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.21078466758102937,
|
|
"grad_norm": 9.993736684687924,
|
|
"learning_rate": 4.815531635768916e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2298,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.2115065328809644,
|
|
"grad_norm": 8.215550634524817,
|
|
"learning_rate": 4.813149246145946e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2413,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.21222839818089945,
|
|
"grad_norm": 6.880787558392017,
|
|
"learning_rate": 4.810752167961971e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2364,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.21295026348083448,
|
|
"grad_norm": 8.022287212940531,
|
|
"learning_rate": 4.808340416438505e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2212,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.21367212878076952,
|
|
"grad_norm": 10.935811215207599,
|
|
"learning_rate": 4.805914006890234e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2454,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.21439399408070453,
|
|
"grad_norm": 7.928137164090748,
|
|
"learning_rate": 4.803472954724928e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2296,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.21511585938063957,
|
|
"grad_norm": 10.309732813070605,
|
|
"learning_rate": 4.801017275443331e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2371,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.2158377246805746,
|
|
"grad_norm": 9.479637952138695,
|
|
"learning_rate": 4.798546984639076e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.21655958998050964,
|
|
"grad_norm": 11.653196066334537,
|
|
"learning_rate": 4.796062097998578e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.21728145528044468,
|
|
"grad_norm": 12.258514957096317,
|
|
"learning_rate": 4.793562631300932e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2292,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.2180033205803797,
|
|
"grad_norm": 7.907531921224019,
|
|
"learning_rate": 4.791048600417824e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2244,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.21872518588031473,
|
|
"grad_norm": 8.125306615010935,
|
|
"learning_rate": 4.788520021313416e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.21944705118024976,
|
|
"grad_norm": 9.119966563903791,
|
|
"learning_rate": 4.785976910044255e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2032,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.2201689164801848,
|
|
"grad_norm": 12.1919687251649,
|
|
"learning_rate": 4.783419282759168e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2312,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.22089078178011984,
|
|
"grad_norm": 10.966337886877598,
|
|
"learning_rate": 4.780847155699157e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2282,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.22161264708005488,
|
|
"grad_norm": 10.89546424385192,
|
|
"learning_rate": 4.778260545197301e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2326,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.22233451237998988,
|
|
"grad_norm": 8.635404642662332,
|
|
"learning_rate": 4.775659467678645e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.22305637767992492,
|
|
"grad_norm": 13.172555524642066,
|
|
"learning_rate": 4.773043939660105e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2527,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.22377824297985996,
|
|
"grad_norm": 10.053263971609315,
|
|
"learning_rate": 4.770413977750353e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.224500108279795,
|
|
"grad_norm": 7.080435865876819,
|
|
"learning_rate": 4.7677695986497225e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2285,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.22522197357973003,
|
|
"grad_norm": 8.489071949818994,
|
|
"learning_rate": 4.7651108191500896e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2217,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.22594383887966504,
|
|
"grad_norm": 7.48950614324499,
|
|
"learning_rate": 4.762437656134778e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.22666570417960008,
|
|
"grad_norm": 11.147541103318718,
|
|
"learning_rate": 4.7597501265784466e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.22738756947953512,
|
|
"grad_norm": 9.2136696723812,
|
|
"learning_rate": 4.757048247546982e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2588,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.22810943477947015,
|
|
"grad_norm": 12.83236893491969,
|
|
"learning_rate": 4.7543320361973884e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2172,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.2288313000794052,
|
|
"grad_norm": 9.608775546642143,
|
|
"learning_rate": 4.751601509777683e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.2295531653793402,
|
|
"grad_norm": 8.893320591024583,
|
|
"learning_rate": 4.748856685626784e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2299,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.23027503067927524,
|
|
"grad_norm": 6.813416083172801,
|
|
"learning_rate": 4.7460975811743986e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2321,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.23099689597921028,
|
|
"grad_norm": 11.910753128704322,
|
|
"learning_rate": 4.743324213940917e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.2317187612791453,
|
|
"grad_norm": 10.880683342821822,
|
|
"learning_rate": 4.740536601537295e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.226,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.23244062657908035,
|
|
"grad_norm": 9.387075119803855,
|
|
"learning_rate": 4.73773476166495e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2052,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.2331624918790154,
|
|
"grad_norm": 9.319384117903645,
|
|
"learning_rate": 4.73491871211564e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2334,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.2338843571789504,
|
|
"grad_norm": 7.92062494143796,
|
|
"learning_rate": 4.7320884707713573e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.226,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.23460622247888543,
|
|
"grad_norm": 7.5905378566734445,
|
|
"learning_rate": 4.7292440556042116e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2381,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.23532808777882047,
|
|
"grad_norm": 7.104939281324458,
|
|
"learning_rate": 4.726385484676318e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.2360499530787555,
|
|
"grad_norm": 7.112864388089024,
|
|
"learning_rate": 4.723512776139679e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.218,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.23677181837869055,
|
|
"grad_norm": 9.110259152947808,
|
|
"learning_rate": 4.7206259482360734e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2342,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.23749368367862556,
|
|
"grad_norm": 8.883129926158574,
|
|
"learning_rate": 4.7177250192969364e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2037,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.2382155489785606,
|
|
"grad_norm": 10.573842474018138,
|
|
"learning_rate": 4.714810007743247e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2226,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.23893741427849563,
|
|
"grad_norm": 9.209497924010089,
|
|
"learning_rate": 4.7118809320854077e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2122,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.23965927957843067,
|
|
"grad_norm": 9.372117356581454,
|
|
"learning_rate": 4.7089378109231294e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.2403811448783657,
|
|
"grad_norm": 8.352914263375952,
|
|
"learning_rate": 4.7059806629453116e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2278,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.24110301017830074,
|
|
"grad_norm": 10.512336386108508,
|
|
"learning_rate": 4.7030095069299257e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.24182487547823575,
|
|
"grad_norm": 9.166170654186674,
|
|
"learning_rate": 4.700024361743893e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.2425467407781708,
|
|
"grad_norm": 10.052841673647455,
|
|
"learning_rate": 4.697025246342967e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2115,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.24326860607810583,
|
|
"grad_norm": 12.87931292960993,
|
|
"learning_rate": 4.6940121797716127e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.227,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.24399047137804086,
|
|
"grad_norm": 11.246155735848257,
|
|
"learning_rate": 4.6909851811628853e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.2447123366779759,
|
|
"grad_norm": 8.247097831323758,
|
|
"learning_rate": 4.68794426973831e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2203,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.375,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.2454342019779109,
|
|
"grad_norm": 7.982031142460416,
|
|
"learning_rate": 4.684889464807755e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2141,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.24615606727784595,
|
|
"grad_norm": 8.859057538137234,
|
|
"learning_rate": 4.681820785769317e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2002,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.24687793257778098,
|
|
"grad_norm": 10.507595703830146,
|
|
"learning_rate": 4.678738252109192e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2272,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.24759979787771602,
|
|
"grad_norm": 9.701082860121796,
|
|
"learning_rate": 4.675641883401553e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.24832166317765106,
|
|
"grad_norm": 9.249938002509944,
|
|
"learning_rate": 4.672531699308425e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.2490435284775861,
|
|
"grad_norm": 10.341717167410792,
|
|
"learning_rate": 4.669407719579562e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2035,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.2497653937775211,
|
|
"grad_norm": 8.681196124713722,
|
|
"learning_rate": 4.666269964052322e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2126,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.25048725907745617,
|
|
"grad_norm": 9.212928240257371,
|
|
"learning_rate": 4.6631184526515384e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2196,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.2512091243773912,
|
|
"grad_norm": 9.229554206053074,
|
|
"learning_rate": 4.6599532053893936e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2172,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.2519309896773262,
|
|
"grad_norm": 7.2543396832627876,
|
|
"learning_rate": 4.6567742423652955e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2158,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.25265285497726125,
|
|
"grad_norm": 8.156746365270518,
|
|
"learning_rate": 4.6535815837657456e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.25337472027719626,
|
|
"grad_norm": 9.476042547482471,
|
|
"learning_rate": 4.6503752498642133e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2247,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.25409658557713133,
|
|
"grad_norm": 11.068802868497535,
|
|
"learning_rate": 4.6471552610210073e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2236,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.25481845087706634,
|
|
"grad_norm": 9.184379291366328,
|
|
"learning_rate": 4.6439216376831447e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2076,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.25554031617700135,
|
|
"grad_norm": 12.322007532110128,
|
|
"learning_rate": 4.6406744003842213e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2124,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.2562621814769364,
|
|
"grad_norm": 7.159317606243659,
|
|
"learning_rate": 4.6374135697442833e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2098,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.2569840467768714,
|
|
"grad_norm": 29.4711769624419,
|
|
"learning_rate": 4.634139166469695e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2267,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.2577059120768065,
|
|
"grad_norm": 7.80532559868148,
|
|
"learning_rate": 4.6308512113530063e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2187,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.0,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.2584277773767415,
|
|
"grad_norm": 7.185706232473536,
|
|
"learning_rate": 4.627549725272822e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.2591496426766765,
|
|
"grad_norm": 7.836489814314065,
|
|
"learning_rate": 4.62423472919367e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2143,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.25987150797661157,
|
|
"grad_norm": 10.312658330041376,
|
|
"learning_rate": 4.6209062441658654e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2176,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.2605933732765466,
|
|
"grad_norm": 7.660185299883346,
|
|
"learning_rate": 4.6175642913253783e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.26131523857648165,
|
|
"grad_norm": 9.630986662907384,
|
|
"learning_rate": 4.6142088918937016e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.26203710387641665,
|
|
"grad_norm": 7.9637824352647195,
|
|
"learning_rate": 4.6108400671777135e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.2627589691763517,
|
|
"grad_norm": 10.302801071286787,
|
|
"learning_rate": 4.607457838569544e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.208,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.26348083447628673,
|
|
"grad_norm": 9.200762753858287,
|
|
"learning_rate": 4.6040622275464355e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.26420269977622174,
|
|
"grad_norm": 8.72614280680941,
|
|
"learning_rate": 4.6006532556706124e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2263,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.2649245650761568,
|
|
"grad_norm": 9.95080301383368,
|
|
"learning_rate": 4.5972309445891386e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2212,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.2656464303760918,
|
|
"grad_norm": 7.898181548134635,
|
|
"learning_rate": 4.593795316033783e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2504,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.2663682956760269,
|
|
"grad_norm": 9.27071552978421,
|
|
"learning_rate": 4.5903463918208815e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2182,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.2670901609759619,
|
|
"grad_norm": 7.471297940396982,
|
|
"learning_rate": 4.586884193851197e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.2678120262758969,
|
|
"grad_norm": 9.198139304204727,
|
|
"learning_rate": 4.5834087441097806e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1971,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.26853389157583196,
|
|
"grad_norm": 7.074538199167009,
|
|
"learning_rate": 4.5799200646658345e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.26925575687576697,
|
|
"grad_norm": 7.9353245918675,
|
|
"learning_rate": 4.576418177672568e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2194,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.26997762217570204,
|
|
"grad_norm": 7.362232757163594,
|
|
"learning_rate": 4.5729031053670596e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2025,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.27069948747563705,
|
|
"grad_norm": 9.803377167019393,
|
|
"learning_rate": 4.569374870070114e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.27142135277557206,
|
|
"grad_norm": 6.923819353668018,
|
|
"learning_rate": 4.565833494186122e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1945,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.2721432180755071,
|
|
"grad_norm": 7.409462373704994,
|
|
"learning_rate": 4.562279000202919e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.27286508337544213,
|
|
"grad_norm": 8.55425171800717,
|
|
"learning_rate": 4.5587114106916366e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.2735869486753772,
|
|
"grad_norm": 8.283760196639793,
|
|
"learning_rate": 4.5551307483065664e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.2743088139753122,
|
|
"grad_norm": 9.334259623082545,
|
|
"learning_rate": 4.5515370357850136e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.2750306792752472,
|
|
"grad_norm": 8.790752805238721,
|
|
"learning_rate": 4.547930295947151e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.2757525445751823,
|
|
"grad_norm": 11.07615206391536,
|
|
"learning_rate": 4.5443105516958737e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.2764744098751173,
|
|
"grad_norm": 12.156443108793454,
|
|
"learning_rate": 4.5406778260166586e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2064,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.27719627517505235,
|
|
"grad_norm": 9.054451472292419,
|
|
"learning_rate": 4.5370321419774117e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2138,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.27791814047498736,
|
|
"grad_norm": 6.698493216999741,
|
|
"learning_rate": 4.5333735227283274e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.2786400057749224,
|
|
"grad_norm": 9.735985205726063,
|
|
"learning_rate": 4.5297019915017375e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1875,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.27936187107485744,
|
|
"grad_norm": 8.810687694126111,
|
|
"learning_rate": 4.5260175716119655e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.212,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.28008373637479245,
|
|
"grad_norm": 10.6784883112821,
|
|
"learning_rate": 4.522320286455179e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.2808056016747275,
|
|
"grad_norm": 9.94830452756906,
|
|
"learning_rate": 4.5186101595092403e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.2815274669746625,
|
|
"grad_norm": 9.418299320110647,
|
|
"learning_rate": 4.5148872143335566e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.2822493322745976,
|
|
"grad_norm": 11.904953591713511,
|
|
"learning_rate": 4.5111514745689307e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.2829711975745326,
|
|
"grad_norm": 11.15522188195295,
|
|
"learning_rate": 4.507402963937414e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.2836930628744676,
|
|
"grad_norm": 11.36508327799745,
|
|
"learning_rate": 4.5036417062421506e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2226,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.28441492817440267,
|
|
"grad_norm": 9.897929517208977,
|
|
"learning_rate": 4.4998677253672297e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2208,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.2851367934743377,
|
|
"grad_norm": 9.778813209573705,
|
|
"learning_rate": 4.496081045277533e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.28585865877427274,
|
|
"grad_norm": 7.8313055942532275,
|
|
"learning_rate": 4.492281690018583e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -9.125,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.28658052407420775,
|
|
"grad_norm": 8.317207474938273,
|
|
"learning_rate": 4.4884696837163905e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2117,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.28730238937414276,
|
|
"grad_norm": 10.26976433894447,
|
|
"learning_rate": 4.484645050577299e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2079,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.28802425467407783,
|
|
"grad_norm": 11.840261029137109,
|
|
"learning_rate": 4.480807814887833e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.214,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.28874611997401284,
|
|
"grad_norm": 8.811651662253073,
|
|
"learning_rate": 4.476958001014544e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2135,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.2894679852739479,
|
|
"grad_norm": 8.267204182411973,
|
|
"learning_rate": 4.4730956334038565e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.2901898505738829,
|
|
"grad_norm": 7.77369954894956,
|
|
"learning_rate": 4.46922073658191e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.2909117158738179,
|
|
"grad_norm": 8.886847365802758,
|
|
"learning_rate": 4.465333335154406e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2115,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.291633581173753,
|
|
"grad_norm": 11.540304268095925,
|
|
"learning_rate": 4.461433453806449e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.292355446473688,
|
|
"grad_norm": 11.614291076457404,
|
|
"learning_rate": 4.457521117302392e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.29307731177362306,
|
|
"grad_norm": 11.959955939062517,
|
|
"learning_rate": 4.45359635048568e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.29379917707355807,
|
|
"grad_norm": 10.625522954210599,
|
|
"learning_rate": 4.4496591782786883e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.2562,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.2945210423734931,
|
|
"grad_norm": 8.706370989411305,
|
|
"learning_rate": 4.44570962568257e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2059,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.29524290767342815,
|
|
"grad_norm": 10.084781546011934,
|
|
"learning_rate": 4.4417477177770905e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2059,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.29596477297336315,
|
|
"grad_norm": 10.211730254994997,
|
|
"learning_rate": 4.4377734797204747e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.2966866382732982,
|
|
"grad_norm": 8.821514902632222,
|
|
"learning_rate": 4.433786936749241e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2048,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.29740850357323323,
|
|
"grad_norm": 9.210606252468882,
|
|
"learning_rate": 4.429788114178049e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.2981303688731683,
|
|
"grad_norm": 8.00555161785549,
|
|
"learning_rate": 4.425777037399529e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1886,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.2988522341731033,
|
|
"grad_norm": 11.207346805964308,
|
|
"learning_rate": 4.42175373188413e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.2995740994730383,
|
|
"grad_norm": 8.172701676742552,
|
|
"learning_rate": 4.4177182231799513e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2072,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.3002959647729734,
|
|
"grad_norm": 5.801373290887345,
|
|
"learning_rate": 4.413670536912584e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.3010178300729084,
|
|
"grad_norm": 6.307391224443462,
|
|
"learning_rate": 4.4096106987849457e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.30173969537284345,
|
|
"grad_norm": 11.867957943680011,
|
|
"learning_rate": 4.405538734577121e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.30246156067277846,
|
|
"grad_norm": 9.987667285927838,
|
|
"learning_rate": 4.401454670146193e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2163,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.30318342597271347,
|
|
"grad_norm": 8.977809465407699,
|
|
"learning_rate": 4.3973585314260836e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1906,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.30390529127264854,
|
|
"grad_norm": 10.825945402642533,
|
|
"learning_rate": 4.393250344427385e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.30462715657258355,
|
|
"grad_norm": 8.126172608631684,
|
|
"learning_rate": 4.389130135237196e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2207,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.3053490218725186,
|
|
"grad_norm": 7.626491220106016,
|
|
"learning_rate": 4.38499793001896e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.3060708871724536,
|
|
"grad_norm": 9.904614258788264,
|
|
"learning_rate": 4.3808537550122913e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.211,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.30679275247238863,
|
|
"grad_norm": 9.163789602486757,
|
|
"learning_rate": 4.376697636532816e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.3075146177723237,
|
|
"grad_norm": 9.658094123529755,
|
|
"learning_rate": 4.3725296009719985e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.3082364830722587,
|
|
"grad_norm": 6.663743741837934,
|
|
"learning_rate": 4.3683496747969804e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1913,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.30895834837219377,
|
|
"grad_norm": 9.431783701004399,
|
|
"learning_rate": 4.364157884550406e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.2088,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.3096802136721288,
|
|
"grad_norm": 10.281531783875531,
|
|
"learning_rate": 4.359954256850259e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.3104020789720638,
|
|
"grad_norm": 9.63227220127926,
|
|
"learning_rate": 4.35573881838969e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.31112394427199885,
|
|
"grad_norm": 10.289092693608229,
|
|
"learning_rate": 4.3515115959368476e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.31184580957193386,
|
|
"grad_norm": 10.974607276803276,
|
|
"learning_rate": 4.3472726163347116e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2179,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.31256767487186893,
|
|
"grad_norm": 6.412992045987879,
|
|
"learning_rate": 4.3430219065009177e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.31328954017180394,
|
|
"grad_norm": 10.403220300532743,
|
|
"learning_rate": 4.3387594934275896e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.31401140547173895,
|
|
"grad_norm": 6.73518127809711,
|
|
"learning_rate": 4.334485404181167e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.314733270771674,
|
|
"grad_norm": 10.054893296825982,
|
|
"learning_rate": 4.3301996659022334e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2176,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.315455136071609,
|
|
"grad_norm": 7.828814625819192,
|
|
"learning_rate": 4.3259023058053444e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2147,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.3161770013715441,
|
|
"grad_norm": 7.009502435156667,
|
|
"learning_rate": 4.3215933511788544e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2081,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.3168988666714791,
|
|
"grad_norm": 9.3284353021479,
|
|
"learning_rate": 4.317272829384743e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.31762073197141416,
|
|
"grad_norm": 8.811880524775491,
|
|
"learning_rate": 4.3129407678584414e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.31834259727134917,
|
|
"grad_norm": 8.125525464636453,
|
|
"learning_rate": 4.3085971941086585e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1915,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.3190644625712842,
|
|
"grad_norm": 8.751734979718838,
|
|
"learning_rate": 4.3042421357172076e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.31978632787121924,
|
|
"grad_norm": 7.608014094617588,
|
|
"learning_rate": 4.2998756203388285e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2152,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.32050819317115425,
|
|
"grad_norm": 7.639743064757877,
|
|
"learning_rate": 4.2954976757010133e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.3212300584710893,
|
|
"grad_norm": 11.460888477745158,
|
|
"learning_rate": 4.2911083296038285e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.32195192377102433,
|
|
"grad_norm": 9.224352773153688,
|
|
"learning_rate": 4.2867076099197446e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1895,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.32267378907095934,
|
|
"grad_norm": 9.244876887068662,
|
|
"learning_rate": 4.2822955445934505e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1772,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.3233956543708944,
|
|
"grad_norm": 8.906087322512382,
|
|
"learning_rate": 4.277872161641681e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.3241175196708294,
|
|
"grad_norm": 9.689414725700667,
|
|
"learning_rate": 4.273437489153041e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.3248393849707645,
|
|
"grad_norm": 8.996613366246144,
|
|
"learning_rate": 4.2689915552878207e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.3255612502706995,
|
|
"grad_norm": 9.607472232534843,
|
|
"learning_rate": 4.2645343882778215e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.3262831155706345,
|
|
"grad_norm": 8.846494209878479,
|
|
"learning_rate": 4.260066016426177e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.32700498087056956,
|
|
"grad_norm": 11.245824423136213,
|
|
"learning_rate": 4.25558646810717e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1856,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.32772684617050457,
|
|
"grad_norm": 8.60581993865266,
|
|
"learning_rate": 4.251095771766055e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.177,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.32844871147043964,
|
|
"grad_norm": 7.6110846227784625,
|
|
"learning_rate": 4.246593955918877e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2188,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.32917057677037465,
|
|
"grad_norm": 7.571006600020749,
|
|
"learning_rate": 4.2420810491522896e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2004,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.32989244207030965,
|
|
"grad_norm": 9.619717180276336,
|
|
"learning_rate": 4.237557080123373e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2037,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.3306143073702447,
|
|
"grad_norm": 9.813397994219041,
|
|
"learning_rate": 4.2330220775594567e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1992,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.33133617267017973,
|
|
"grad_norm": 8.765749359430536,
|
|
"learning_rate": 4.228476070257929e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.3320580379701148,
|
|
"grad_norm": 10.82246666876482,
|
|
"learning_rate": 4.223919087086062e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1802,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.3327799032700498,
|
|
"grad_norm": 7.941514770204173,
|
|
"learning_rate": 4.2193511569808225e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1991,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.3335017685699848,
|
|
"grad_norm": 7.912745662185516,
|
|
"learning_rate": 4.214772308948693e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.3342236338699199,
|
|
"grad_norm": 10.9521651772932,
|
|
"learning_rate": 4.2101825720654827e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2081,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.3349454991698549,
|
|
"grad_norm": 6.99023599526195,
|
|
"learning_rate": 4.205581975476146e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2041,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.33566736446978995,
|
|
"grad_norm": 8.603344530140768,
|
|
"learning_rate": 4.200970548394598e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1965,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.33638922976972496,
|
|
"grad_norm": 9.717823679934542,
|
|
"learning_rate": 4.196348320103527e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.33711109506966,
|
|
"grad_norm": 7.935814772911749,
|
|
"learning_rate": 4.191715319954209e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.33783296036959504,
|
|
"grad_norm": 8.769171773282734,
|
|
"learning_rate": 4.187071577366321e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.33855482566953005,
|
|
"grad_norm": 7.096843320580391,
|
|
"learning_rate": 4.182417121827755e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1906,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.3392766909694651,
|
|
"grad_norm": 10.058671513695627,
|
|
"learning_rate": 4.177751982894433e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.1903,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.3399985562694001,
|
|
"grad_norm": 10.565296750226013,
|
|
"learning_rate": 4.1730761901901135e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1993,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.3407204215693352,
|
|
"grad_norm": 9.850412180420557,
|
|
"learning_rate": 4.168389773406209e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.3414422868692702,
|
|
"grad_norm": 11.906676776121467,
|
|
"learning_rate": 4.1636927623015927e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.3421641521692052,
|
|
"grad_norm": 10.205674489245993,
|
|
"learning_rate": 4.158985186702415e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2092,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.34288601746914027,
|
|
"grad_norm": 9.260984330253814,
|
|
"learning_rate": 4.1542670765019104e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.3436078827690753,
|
|
"grad_norm": 8.949027397431555,
|
|
"learning_rate": 4.149538461660206e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.34432974806901034,
|
|
"grad_norm": 6.720014771416349,
|
|
"learning_rate": 4.144799372204136e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1826,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.34505161336894535,
|
|
"grad_norm": 8.959024936705582,
|
|
"learning_rate": 4.140049838227049e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1987,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.34577347866888036,
|
|
"grad_norm": 10.372748362230645,
|
|
"learning_rate": 4.135289889888615e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.34649534396881543,
|
|
"grad_norm": 8.270570715124181,
|
|
"learning_rate": 4.130519557414636e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.34721720926875044,
|
|
"grad_norm": 9.893879868420212,
|
|
"learning_rate": 4.1257388710968533e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1925,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.3479390745686855,
|
|
"grad_norm": 12.331591426723168,
|
|
"learning_rate": 4.120947861292757e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.3486609398686205,
|
|
"grad_norm": 10.172376095851071,
|
|
"learning_rate": 4.11614655842539e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.3493828051685555,
|
|
"grad_norm": 11.310294528639917,
|
|
"learning_rate": 4.111334992983156e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1973,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.3501046704684906,
|
|
"grad_norm": 9.138376308040302,
|
|
"learning_rate": 4.1065131955196294e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.3508265357684256,
|
|
"grad_norm": 10.948111955434124,
|
|
"learning_rate": 4.1016811966533536e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1823,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.35154840106836066,
|
|
"grad_norm": 6.8483776654650965,
|
|
"learning_rate": 4.096839027067656e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2069,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.35227026636829567,
|
|
"grad_norm": 7.262308675444548,
|
|
"learning_rate": 4.0919867175104435e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.35299213166823074,
|
|
"grad_norm": 17.175486759838638,
|
|
"learning_rate": 4.0871242987940155e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1981,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.35371399696816574,
|
|
"grad_norm": 6.418913714694265,
|
|
"learning_rate": 4.082251801794865e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.35443586226810075,
|
|
"grad_norm": 6.8125539564399515,
|
|
"learning_rate": 4.0773692574534795e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1871,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.3551577275680358,
|
|
"grad_norm": 7.814190613362822,
|
|
"learning_rate": 4.072476696774151e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1852,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.35587959286797083,
|
|
"grad_norm": 10.826610307359978,
|
|
"learning_rate": 4.0675741508247715e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.3566014581679059,
|
|
"grad_norm": 8.596687269801581,
|
|
"learning_rate": 4.062661650736643e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1869,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.3573233234678409,
|
|
"grad_norm": 6.416195679478135,
|
|
"learning_rate": 4.0577392277042766e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1855,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.3580451887677759,
|
|
"grad_norm": 8.918928643083534,
|
|
"learning_rate": 4.0528069129851914e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1846,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.358767054067711,
|
|
"grad_norm": 8.47545192360217,
|
|
"learning_rate": 4.0478647378997223e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2025,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.359488919367646,
|
|
"grad_norm": 8.09646150016637,
|
|
"learning_rate": 4.0429127338308154e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.206,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.36021078466758105,
|
|
"grad_norm": 7.912242395511905,
|
|
"learning_rate": 4.037950932223832e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.36093264996751606,
|
|
"grad_norm": 8.510565195808283,
|
|
"learning_rate": 4.032979364586349e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2007,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.36093264996751606,
|
|
"eval_logits/chosen": -2.90625,
|
|
"eval_logits/rejected": -2.65625,
|
|
"eval_logps/chosen": -700.0,
|
|
"eval_logps/rejected": -1040.0,
|
|
"eval_loss": 0.22352416813373566,
|
|
"eval_rewards/accuracies": 0.9081980586051941,
|
|
"eval_rewards/chosen": -5.3125,
|
|
"eval_rewards/margins": 3.546875,
|
|
"eval_rewards/rejected": -8.875,
|
|
"eval_runtime": 3594.5422,
|
|
"eval_samples_per_second": 27.403,
|
|
"eval_steps_per_second": 0.428,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.36165451526745107,
|
|
"grad_norm": 9.508216962196549,
|
|
"learning_rate": 4.027998062487955e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"epoch": 0.36237638056738614,
|
|
"grad_norm": 11.433083041803624,
|
|
"learning_rate": 4.023007057560057e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"epoch": 0.36309824586732115,
|
|
"grad_norm": 9.561775751013597,
|
|
"learning_rate": 4.018006381495671e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1764,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"epoch": 0.3638201111672562,
|
|
"grad_norm": 8.343880295640096,
|
|
"learning_rate": 4.012996066049229e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1988,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"epoch": 0.3645419764671912,
|
|
"grad_norm": 7.788342844437915,
|
|
"learning_rate": 4.0079761430363676e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1751,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"epoch": 0.36526384176712623,
|
|
"grad_norm": 9.014324627097576,
|
|
"learning_rate": 4.002946644333739e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"epoch": 0.3659857070670613,
|
|
"grad_norm": 10.445201108593542,
|
|
"learning_rate": 3.997907601878796e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1911,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"epoch": 0.3667075723669963,
|
|
"grad_norm": 8.250001844997403,
|
|
"learning_rate": 3.992859047669596e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"epoch": 0.36742943766693137,
|
|
"grad_norm": 11.753669122881204,
|
|
"learning_rate": 3.987801013764596e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1879,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"epoch": 0.3681513029668664,
|
|
"grad_norm": 12.658837897492882,
|
|
"learning_rate": 3.9827335322824496e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"epoch": 0.3688731682668014,
|
|
"grad_norm": 7.844631820767091,
|
|
"learning_rate": 3.977656635401805e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"epoch": 0.36959503356673645,
|
|
"grad_norm": 7.842154920577124,
|
|
"learning_rate": 3.972570355361093e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"epoch": 0.37031689886667146,
|
|
"grad_norm": 12.429197801825422,
|
|
"learning_rate": 3.967474724458334e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"epoch": 0.3710387641666065,
|
|
"grad_norm": 9.022738866247183,
|
|
"learning_rate": 3.962369775050922e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"epoch": 0.37176062946654154,
|
|
"grad_norm": 7.735894816170176,
|
|
"learning_rate": 3.957255539555426e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"epoch": 0.3724824947664766,
|
|
"grad_norm": 7.904648881093523,
|
|
"learning_rate": 3.9521320504473804e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1898,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"epoch": 0.3732043600664116,
|
|
"grad_norm": 8.068500078950347,
|
|
"learning_rate": 3.9469993402610823e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"epoch": 0.3739262253663466,
|
|
"grad_norm": 9.258484145773922,
|
|
"learning_rate": 3.941857441589382e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"epoch": 0.3746480906662817,
|
|
"grad_norm": 9.720892187785136,
|
|
"learning_rate": 3.936706387083476e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"epoch": 0.3753699559662167,
|
|
"grad_norm": 5.5288559635381365,
|
|
"learning_rate": 3.9315462094527006e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.37609182126615176,
|
|
"grad_norm": 7.613626077988378,
|
|
"learning_rate": 3.9263769414643273e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1846,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"epoch": 0.37681368656608677,
|
|
"grad_norm": 6.807486326402407,
|
|
"learning_rate": 3.9211986159433477e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"epoch": 0.3775355518660218,
|
|
"grad_norm": 9.160049813590764,
|
|
"learning_rate": 3.91601126577227e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1827,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"epoch": 0.37825741716595684,
|
|
"grad_norm": 10.144956306040507,
|
|
"learning_rate": 3.910814923890911e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1734,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"epoch": 0.37897928246589185,
|
|
"grad_norm": 10.789604509188436,
|
|
"learning_rate": 3.9056096232961834e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1953,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"epoch": 0.3797011477658269,
|
|
"grad_norm": 8.03159337115462,
|
|
"learning_rate": 3.9003953970418877e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1836,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"epoch": 0.38042301306576193,
|
|
"grad_norm": 9.599822536896923,
|
|
"learning_rate": 3.8951722782385046e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1911,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"epoch": 0.38114487836569694,
|
|
"grad_norm": 10.071856627849183,
|
|
"learning_rate": 3.8899403000529803e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"epoch": 0.381866743665632,
|
|
"grad_norm": 10.524225607275767,
|
|
"learning_rate": 3.8846994957085194e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"epoch": 0.382588608965567,
|
|
"grad_norm": 8.616712400688618,
|
|
"learning_rate": 3.879449898484374e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1873,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"epoch": 0.3833104742655021,
|
|
"grad_norm": 8.45520493106579,
|
|
"learning_rate": 3.874191541715629e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1777,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"epoch": 0.3840323395654371,
|
|
"grad_norm": 6.786490880185805,
|
|
"learning_rate": 3.868924458792994e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"epoch": 0.3847542048653721,
|
|
"grad_norm": 9.306230356711307,
|
|
"learning_rate": 3.86364868316259e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1961,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"epoch": 0.38547607016530716,
|
|
"grad_norm": 6.874539829983695,
|
|
"learning_rate": 3.8583642483257374e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"epoch": 0.38619793546524217,
|
|
"grad_norm": 9.535981769729352,
|
|
"learning_rate": 3.85307118783874e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"epoch": 0.38691980076517724,
|
|
"grad_norm": 7.916792530196583,
|
|
"learning_rate": 3.8477695353126785e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1841,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"epoch": 0.38764166606511224,
|
|
"grad_norm": 11.021130141780702,
|
|
"learning_rate": 3.8424593244131897e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1947,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"epoch": 0.3883635313650473,
|
|
"grad_norm": 8.203256258991978,
|
|
"learning_rate": 3.83714058886026e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"epoch": 0.3890853966649823,
|
|
"grad_norm": 7.923518332280399,
|
|
"learning_rate": 3.8318133624280046e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"epoch": 0.38980726196491733,
|
|
"grad_norm": 13.720539493582635,
|
|
"learning_rate": 3.826477678944457e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1945,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.3905291272648524,
|
|
"grad_norm": 7.1393388118343495,
|
|
"learning_rate": 3.821133572291354e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1814,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"epoch": 0.3912509925647874,
|
|
"grad_norm": 9.329246298724282,
|
|
"learning_rate": 3.8157810764039187e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1938,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"epoch": 0.39197285786472247,
|
|
"grad_norm": 6.811570201519407,
|
|
"learning_rate": 3.810420225270647e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"epoch": 0.3926947231646575,
|
|
"grad_norm": 6.663922484272897,
|
|
"learning_rate": 3.80505105293309e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1847,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"epoch": 0.3934165884645925,
|
|
"grad_norm": 9.592577634918783,
|
|
"learning_rate": 3.799673593485638e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1773,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"epoch": 0.39413845376452755,
|
|
"grad_norm": 9.134235793523999,
|
|
"learning_rate": 3.794287881075307e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1741,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"epoch": 0.39486031906446256,
|
|
"grad_norm": 8.925533209727694,
|
|
"learning_rate": 3.788893949901517e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"epoch": 0.3955821843643976,
|
|
"grad_norm": 7.220703628171884,
|
|
"learning_rate": 3.783491834215879e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"epoch": 0.39630404966433264,
|
|
"grad_norm": 10.56460893328502,
|
|
"learning_rate": 3.778081568321976e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"epoch": 0.39702591496426765,
|
|
"grad_norm": 10.362564107960075,
|
|
"learning_rate": 3.772663186575143e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 0.3977477802642027,
|
|
"grad_norm": 9.293115038985665,
|
|
"learning_rate": 3.7672367233822537e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"epoch": 0.3984696455641377,
|
|
"grad_norm": 8.820973054087007,
|
|
"learning_rate": 3.761802213201498e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"epoch": 0.3991915108640728,
|
|
"grad_norm": 8.927273915289893,
|
|
"learning_rate": 3.756359690542163e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"epoch": 0.3999133761640078,
|
|
"grad_norm": 9.274501424450483,
|
|
"learning_rate": 3.750909189964417e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"epoch": 0.4006352414639428,
|
|
"grad_norm": 14.27324930632179,
|
|
"learning_rate": 3.7454507460790893e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"epoch": 0.40135710676387787,
|
|
"grad_norm": 5.934956865301077,
|
|
"learning_rate": 3.739984393547446e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1771,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"epoch": 0.4020789720638129,
|
|
"grad_norm": 8.744047306639088,
|
|
"learning_rate": 3.7345101670809755e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"epoch": 0.40280083736374794,
|
|
"grad_norm": 10.141402293967214,
|
|
"learning_rate": 3.7290281014411655e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1761,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"epoch": 0.40352270266368295,
|
|
"grad_norm": 8.951801438600679,
|
|
"learning_rate": 3.723538231439284e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"epoch": 0.40424456796361796,
|
|
"grad_norm": 9.18772270207208,
|
|
"learning_rate": 3.7180405919361547e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1996,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.404966433263553,
|
|
"grad_norm": 11.802072092894162,
|
|
"learning_rate": 3.7125352178419396e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"epoch": 0.40568829856348804,
|
|
"grad_norm": 7.441092805572635,
|
|
"learning_rate": 3.707022144115914e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1979,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"epoch": 0.4064101638634231,
|
|
"grad_norm": 8.542112860478404,
|
|
"learning_rate": 3.701501405766248e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1833,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"epoch": 0.4071320291633581,
|
|
"grad_norm": 8.690977859423688,
|
|
"learning_rate": 3.69597303784978e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"epoch": 0.4078538944632932,
|
|
"grad_norm": 7.133375992414793,
|
|
"learning_rate": 3.690437075471797e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1834,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.0,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"epoch": 0.4085757597632282,
|
|
"grad_norm": 9.060343279065481,
|
|
"learning_rate": 3.684893553785815e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"epoch": 0.4092976250631632,
|
|
"grad_norm": 9.150169701262556,
|
|
"learning_rate": 3.6793425079933446e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"epoch": 0.41001949036309826,
|
|
"grad_norm": 11.459154584979037,
|
|
"learning_rate": 3.67378397334368e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.191,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"epoch": 0.41074135566303327,
|
|
"grad_norm": 10.628103177908564,
|
|
"learning_rate": 3.668217985133668e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1765,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"epoch": 0.41146322096296833,
|
|
"grad_norm": 10.663063859010725,
|
|
"learning_rate": 3.662644578707486e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"epoch": 0.41218508626290334,
|
|
"grad_norm": 9.68068326501746,
|
|
"learning_rate": 3.657063789456418e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1893,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"epoch": 0.41290695156283835,
|
|
"grad_norm": 9.038765481874343,
|
|
"learning_rate": 3.6514756528186283e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1737,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"epoch": 0.4136288168627734,
|
|
"grad_norm": 7.8137408122782315,
|
|
"learning_rate": 3.645880204278936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"epoch": 0.41435068216270843,
|
|
"grad_norm": 10.519994806474793,
|
|
"learning_rate": 3.640277479368594e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"epoch": 0.4150725474626435,
|
|
"grad_norm": 6.618841094002535,
|
|
"learning_rate": 3.6346675136650595e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1686,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"epoch": 0.4157944127625785,
|
|
"grad_norm": 9.449161368315643,
|
|
"learning_rate": 3.629050342791766e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1883,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"epoch": 0.4165162780625135,
|
|
"grad_norm": 8.645062230868888,
|
|
"learning_rate": 3.6234260024179033e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"epoch": 0.4172381433624486,
|
|
"grad_norm": 8.61829308724476,
|
|
"learning_rate": 3.6177945282581866e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"epoch": 0.4179600086623836,
|
|
"grad_norm": 8.790765287429858,
|
|
"learning_rate": 3.6121559560726313e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"epoch": 0.41868187396231865,
|
|
"grad_norm": 10.485975334565037,
|
|
"learning_rate": 3.6065103216663237e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.187,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.41940373926225366,
|
|
"grad_norm": 7.596154187472179,
|
|
"learning_rate": 3.600857660889199e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"epoch": 0.42012560456218867,
|
|
"grad_norm": 7.510040671882969,
|
|
"learning_rate": 3.5951980096358055e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"epoch": 0.42084746986212374,
|
|
"grad_norm": 6.151781029516534,
|
|
"learning_rate": 3.589531403845085e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1812,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"epoch": 0.42156933516205874,
|
|
"grad_norm": 10.349825384153817,
|
|
"learning_rate": 3.5838578795001393e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"epoch": 0.4222912004619938,
|
|
"grad_norm": 8.687369080835087,
|
|
"learning_rate": 3.578177472628002e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"epoch": 0.4230130657619288,
|
|
"grad_norm": 9.245880790108894,
|
|
"learning_rate": 3.572490219299414e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1933,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"epoch": 0.42373493106186383,
|
|
"grad_norm": 10.513199703384252,
|
|
"learning_rate": 3.5667961556285876e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.172,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"epoch": 0.4244567963617989,
|
|
"grad_norm": 8.067276310939405,
|
|
"learning_rate": 3.561095317772984e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"epoch": 0.4251786616617339,
|
|
"grad_norm": 9.335763045264015,
|
|
"learning_rate": 3.5553877419330797e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"epoch": 0.42590052696166897,
|
|
"grad_norm": 8.536819036424028,
|
|
"learning_rate": 3.5496734643521364e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.198,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"epoch": 0.426622392261604,
|
|
"grad_norm": 10.712349432459751,
|
|
"learning_rate": 3.543952521315975e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1793,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"epoch": 0.42734425756153904,
|
|
"grad_norm": 9.762289940914592,
|
|
"learning_rate": 3.538224949152738e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.179,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"epoch": 0.42806612286147405,
|
|
"grad_norm": 8.279157154276295,
|
|
"learning_rate": 3.5324907842326676e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"epoch": 0.42878798816140906,
|
|
"grad_norm": 7.703443487862446,
|
|
"learning_rate": 3.526750062967866e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"epoch": 0.4295098534613441,
|
|
"grad_norm": 8.915401918749712,
|
|
"learning_rate": 3.5210028218120714e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"epoch": 0.43023171876127914,
|
|
"grad_norm": 7.356754892003317,
|
|
"learning_rate": 3.515249097260422e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1744,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"epoch": 0.4309535840612142,
|
|
"grad_norm": 8.27577072471933,
|
|
"learning_rate": 3.509488925849227e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1813,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"epoch": 0.4316754493611492,
|
|
"grad_norm": 8.91909433201475,
|
|
"learning_rate": 3.503722344155731e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"epoch": 0.4323973146610842,
|
|
"grad_norm": 10.528827951999629,
|
|
"learning_rate": 3.4979493887978864e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"epoch": 0.4331191799610193,
|
|
"grad_norm": 6.577916440055691,
|
|
"learning_rate": 3.4921700964341173e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.4338410452609543,
|
|
"grad_norm": 9.810703167678307,
|
|
"learning_rate": 3.486384503763089e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"epoch": 0.43456291056088936,
|
|
"grad_norm": 7.569455182394066,
|
|
"learning_rate": 3.480592647523472e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"epoch": 0.43528477586082437,
|
|
"grad_norm": 6.998884892266847,
|
|
"learning_rate": 3.4747945644937133e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"epoch": 0.4360066411607594,
|
|
"grad_norm": 7.151254090045432,
|
|
"learning_rate": 3.468990291491799e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1727,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"epoch": 0.43672850646069444,
|
|
"grad_norm": 6.561607683446632,
|
|
"learning_rate": 3.46317986537502e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1876,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"epoch": 0.43745037176062945,
|
|
"grad_norm": 6.7139191061366805,
|
|
"learning_rate": 3.4573633230397414e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"epoch": 0.4381722370605645,
|
|
"grad_norm": 8.411313986797488,
|
|
"learning_rate": 3.451540701421167e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"epoch": 0.4388941023604995,
|
|
"grad_norm": 8.23452700502391,
|
|
"learning_rate": 3.445712037493104e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"epoch": 0.43961596766043454,
|
|
"grad_norm": 7.143620144559741,
|
|
"learning_rate": 3.439877368267725e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1989,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"epoch": 0.4403378329603696,
|
|
"grad_norm": 5.802482964711945,
|
|
"learning_rate": 3.434036730795342e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"epoch": 0.4410596982603046,
|
|
"grad_norm": 8.719522994114646,
|
|
"learning_rate": 3.428190162164162e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1428,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"epoch": 0.4417815635602397,
|
|
"grad_norm": 8.401165390196324,
|
|
"learning_rate": 3.422337699500056e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"epoch": 0.4425034288601747,
|
|
"grad_norm": 7.273765835778647,
|
|
"learning_rate": 3.4164793799663207e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"epoch": 0.44322529416010975,
|
|
"grad_norm": 11.508967977205781,
|
|
"learning_rate": 3.410615240763446e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2057,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"epoch": 0.44394715946004476,
|
|
"grad_norm": 9.492547090583608,
|
|
"learning_rate": 3.4047453191288775e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"epoch": 0.44466902475997977,
|
|
"grad_norm": 6.826894009448305,
|
|
"learning_rate": 3.3988696523367777e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"epoch": 0.44539089005991483,
|
|
"grad_norm": 7.908259059531054,
|
|
"learning_rate": 3.3929882776977905e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1855,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"epoch": 0.44611275535984984,
|
|
"grad_norm": 9.323313899318885,
|
|
"learning_rate": 3.387101232558807e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"epoch": 0.4468346206597849,
|
|
"grad_norm": 10.574159066231,
|
|
"learning_rate": 3.381208554302723e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"epoch": 0.4475564859597199,
|
|
"grad_norm": 9.45659923763495,
|
|
"learning_rate": 3.3753102803482094e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.44827835125965493,
|
|
"grad_norm": 7.8086390499161045,
|
|
"learning_rate": 3.3694064481494655e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"epoch": 0.44900021655959,
|
|
"grad_norm": 6.5084303039612985,
|
|
"learning_rate": 3.3634970951959867e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1641,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"epoch": 0.449722081859525,
|
|
"grad_norm": 8.137911391940417,
|
|
"learning_rate": 3.357582259012327e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1842,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"epoch": 0.45044394715946007,
|
|
"grad_norm": 8.376896629675258,
|
|
"learning_rate": 3.351661977157859e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"epoch": 0.4511658124593951,
|
|
"grad_norm": 10.56542182965613,
|
|
"learning_rate": 3.3457362872265324e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1604,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"epoch": 0.4518876777593301,
|
|
"grad_norm": 7.833165560372883,
|
|
"learning_rate": 3.339805226846642e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"epoch": 0.45260954305926515,
|
|
"grad_norm": 7.8349709497727025,
|
|
"learning_rate": 3.3338688336805823e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"epoch": 0.45333140835920016,
|
|
"grad_norm": 9.092744073048326,
|
|
"learning_rate": 3.327927145424613e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"epoch": 0.4540532736591352,
|
|
"grad_norm": 10.949550588632437,
|
|
"learning_rate": 3.321980199808616e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.2021,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"epoch": 0.45477513895907024,
|
|
"grad_norm": 10.844258468917948,
|
|
"learning_rate": 3.316028034595861e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"epoch": 0.45549700425900524,
|
|
"grad_norm": 13.438072021230731,
|
|
"learning_rate": 3.3100706875827576e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1739,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"epoch": 0.4562188695589403,
|
|
"grad_norm": 10.43903315431466,
|
|
"learning_rate": 3.304108196598624e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"epoch": 0.4569407348588753,
|
|
"grad_norm": 9.193341851899063,
|
|
"learning_rate": 3.29814059950544e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"epoch": 0.4576626001588104,
|
|
"grad_norm": 9.553793370343385,
|
|
"learning_rate": 3.2921679341976104e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"epoch": 0.4583844654587454,
|
|
"grad_norm": 9.073658565124525,
|
|
"learning_rate": 3.286190238601725e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"epoch": 0.4591063307586804,
|
|
"grad_norm": 10.747256513722208,
|
|
"learning_rate": 3.280207550676312e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1773,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"epoch": 0.45982819605861547,
|
|
"grad_norm": 10.072254030204776,
|
|
"learning_rate": 3.274219908411605e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1759,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"epoch": 0.4605500613585505,
|
|
"grad_norm": 9.55913362196861,
|
|
"learning_rate": 3.2682273498292957e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"epoch": 0.46127192665848554,
|
|
"grad_norm": 7.6694528016919055,
|
|
"learning_rate": 3.262229912982295e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"epoch": 0.46199379195842055,
|
|
"grad_norm": 10.685410106690007,
|
|
"learning_rate": 3.2562276359544917e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.4627156572583556,
|
|
"grad_norm": 6.117106649506752,
|
|
"learning_rate": 3.2502205568605087e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.167,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"epoch": 0.4634375225582906,
|
|
"grad_norm": 10.7792422427223,
|
|
"learning_rate": 3.244208713845461e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1907,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"epoch": 0.46415938785822564,
|
|
"grad_norm": 7.813563275474187,
|
|
"learning_rate": 3.2381921450847187e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"epoch": 0.4648812531581607,
|
|
"grad_norm": 10.042020947007074,
|
|
"learning_rate": 3.2321708887836573e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"epoch": 0.4656031184580957,
|
|
"grad_norm": 10.113981129252544,
|
|
"learning_rate": 3.226144983177419e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1802,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"epoch": 0.4663249837580308,
|
|
"grad_norm": 10.153352236033115,
|
|
"learning_rate": 3.220114466530668e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1811,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"epoch": 0.4670468490579658,
|
|
"grad_norm": 10.905089224037248,
|
|
"learning_rate": 3.214079377137352e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1801,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"epoch": 0.4677687143579008,
|
|
"grad_norm": 8.04177832973184,
|
|
"learning_rate": 3.2080397533204526e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"epoch": 0.46849057965783586,
|
|
"grad_norm": 13.034607545233241,
|
|
"learning_rate": 3.201995633431747e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"epoch": 0.46921244495777087,
|
|
"grad_norm": 6.884563442702245,
|
|
"learning_rate": 3.1959470558515613e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 0.46993431025770593,
|
|
"grad_norm": 9.142548647265967,
|
|
"learning_rate": 3.189894058988528e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"epoch": 0.47065617555764094,
|
|
"grad_norm": 10.80278074451447,
|
|
"learning_rate": 3.183836681279344e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"epoch": 0.47137804085757595,
|
|
"grad_norm": 10.130379027496794,
|
|
"learning_rate": 3.177774961188524e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"epoch": 0.472099906157511,
|
|
"grad_norm": 7.542971934363328,
|
|
"learning_rate": 3.171708937208154e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"epoch": 0.472821771457446,
|
|
"grad_norm": 10.844193020610753,
|
|
"learning_rate": 3.1656386478576553e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"epoch": 0.4735436367573811,
|
|
"grad_norm": 8.465995887258453,
|
|
"learning_rate": 3.159564131683529e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"epoch": 0.4742655020573161,
|
|
"grad_norm": 10.357924748793268,
|
|
"learning_rate": 3.15348542725912e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1754,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"epoch": 0.4749873673572511,
|
|
"grad_norm": 8.386237735602531,
|
|
"learning_rate": 3.1474025731843675e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1686,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"epoch": 0.4757092326571862,
|
|
"grad_norm": 7.538386472518327,
|
|
"learning_rate": 3.141315608085561e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"epoch": 0.4764310979571212,
|
|
"grad_norm": 11.830534544208183,
|
|
"learning_rate": 3.1352245706150966e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.47715296325705625,
|
|
"grad_norm": 8.216481532924343,
|
|
"learning_rate": 3.129129499451229e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"epoch": 0.47787482855699126,
|
|
"grad_norm": 8.346007528728483,
|
|
"learning_rate": 3.123030433297823e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1709,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"epoch": 0.4785966938569263,
|
|
"grad_norm": 11.851428689841443,
|
|
"learning_rate": 3.11692741088412e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2184,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"epoch": 0.47931855915686133,
|
|
"grad_norm": 11.616947258566963,
|
|
"learning_rate": 3.110820470964479e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1676,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"epoch": 0.48004042445679634,
|
|
"grad_norm": 8.689762439600946,
|
|
"learning_rate": 3.1047096523181334e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"epoch": 0.4807622897567314,
|
|
"grad_norm": 11.54632495710895,
|
|
"learning_rate": 3.098594993748949e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1905,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"epoch": 0.4814841550566664,
|
|
"grad_norm": 8.85697591328895,
|
|
"learning_rate": 3.092476534085177e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1831,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"epoch": 0.4822060203566015,
|
|
"grad_norm": 6.837080063986226,
|
|
"learning_rate": 3.0863543121792023e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"epoch": 0.4829278856565365,
|
|
"grad_norm": 8.376719148936688,
|
|
"learning_rate": 3.080228366907302e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"epoch": 0.4836497509564715,
|
|
"grad_norm": 11.991657008786461,
|
|
"learning_rate": 3.074098737169395e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"epoch": 0.48437161625640657,
|
|
"grad_norm": 6.801458005281131,
|
|
"learning_rate": 3.0679654618887997e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"epoch": 0.4850934815563416,
|
|
"grad_norm": 9.454920867053952,
|
|
"learning_rate": 3.0618285800119807e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1882,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"epoch": 0.48581534685627664,
|
|
"grad_norm": 8.405586504954512,
|
|
"learning_rate": 3.0556881305083064e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1845,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"epoch": 0.48653721215621165,
|
|
"grad_norm": 9.286913760440065,
|
|
"learning_rate": 3.049544152369798e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1808,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"epoch": 0.48725907745614666,
|
|
"grad_norm": 9.84775376184747,
|
|
"learning_rate": 3.043396684610887e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"epoch": 0.4879809427560817,
|
|
"grad_norm": 9.876467151373804,
|
|
"learning_rate": 3.0372457662681597e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"epoch": 0.48870280805601674,
|
|
"grad_norm": 8.982483288644246,
|
|
"learning_rate": 3.031091436400117e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"epoch": 0.4894246733559518,
|
|
"grad_norm": 8.183235398587287,
|
|
"learning_rate": 3.024933734086922e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"epoch": 0.4901465386558868,
|
|
"grad_norm": 7.608116127645515,
|
|
"learning_rate": 3.018772698430152e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"epoch": 0.4908684039558218,
|
|
"grad_norm": 11.231827143842242,
|
|
"learning_rate": 3.0126083685525526e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1669,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.4915902692557569,
|
|
"grad_norm": 16.428828202202983,
|
|
"learning_rate": 3.006440783597787e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1696,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"epoch": 0.4923121345556919,
|
|
"grad_norm": 11.61196417270313,
|
|
"learning_rate": 3.000269982730189e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1766,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"epoch": 0.49303399985562696,
|
|
"grad_norm": 8.17788758978238,
|
|
"learning_rate": 2.9940960051345135e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"epoch": 0.49375586515556197,
|
|
"grad_norm": 9.07977507120006,
|
|
"learning_rate": 2.987918890015685e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"epoch": 0.494477730455497,
|
|
"grad_norm": 13.691630630266364,
|
|
"learning_rate": 2.981738676598555e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"epoch": 0.49519959575543204,
|
|
"grad_norm": 10.993076123812866,
|
|
"learning_rate": 2.9755554041276465e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1889,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"epoch": 0.49592146105536705,
|
|
"grad_norm": 11.646557564452921,
|
|
"learning_rate": 2.9693691118669117e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1677,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"epoch": 0.4966433263553021,
|
|
"grad_norm": 8.874925129899568,
|
|
"learning_rate": 2.9631798390994726e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1841,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"epoch": 0.4973651916552371,
|
|
"grad_norm": 8.267348409663557,
|
|
"learning_rate": 2.9569876251273826e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1684,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"epoch": 0.4980870569551722,
|
|
"grad_norm": 11.307205545557014,
|
|
"learning_rate": 2.9507925092713685e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1592,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"epoch": 0.4988089222551072,
|
|
"grad_norm": 9.259383366590928,
|
|
"learning_rate": 2.9445945308705866e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1956,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"epoch": 0.4995307875550422,
|
|
"grad_norm": 7.91332378148377,
|
|
"learning_rate": 2.938393729282369e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"epoch": 0.5002526528549772,
|
|
"grad_norm": 9.581050058932107,
|
|
"learning_rate": 2.932190143881976e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.165,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"epoch": 0.5009745181549123,
|
|
"grad_norm": 11.548026800858397,
|
|
"learning_rate": 2.925983814062342e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"epoch": 0.5016963834548473,
|
|
"grad_norm": 9.32142861680438,
|
|
"learning_rate": 2.9197747792338344e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1701,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"epoch": 0.5024182487547824,
|
|
"grad_norm": 8.751197433738259,
|
|
"learning_rate": 2.9135630788239936e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"epoch": 0.5031401140547174,
|
|
"grad_norm": 9.093507158463877,
|
|
"learning_rate": 2.907348752277286e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"epoch": 0.5038619793546524,
|
|
"grad_norm": 10.072538127620183,
|
|
"learning_rate": 2.901131839054856e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1771,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"epoch": 0.5045838446545875,
|
|
"grad_norm": 7.9225853596575515,
|
|
"learning_rate": 2.894912378634272e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"epoch": 0.5053057099545225,
|
|
"grad_norm": 8.508668273373068,
|
|
"learning_rate": 2.888690410509278e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1732,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.5060275752544575,
|
|
"grad_norm": 9.683917887834271,
|
|
"learning_rate": 2.8824659741895424e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"epoch": 0.5067494405543925,
|
|
"grad_norm": 11.888118286993855,
|
|
"learning_rate": 2.8762391092004054e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1856,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"epoch": 0.5074713058543275,
|
|
"grad_norm": 10.520962949102882,
|
|
"learning_rate": 2.8700098550826317e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1766,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"epoch": 0.5081931711542627,
|
|
"grad_norm": 6.050995114285996,
|
|
"learning_rate": 2.8637782513921537e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"epoch": 0.5089150364541977,
|
|
"grad_norm": 11.386068773907736,
|
|
"learning_rate": 2.857544337699826e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"epoch": 0.5096369017541327,
|
|
"grad_norm": 9.512039437649568,
|
|
"learning_rate": 2.85130815359117e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.17,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"epoch": 0.5103587670540677,
|
|
"grad_norm": 11.114854171347922,
|
|
"learning_rate": 2.845069738666128e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"epoch": 0.5110806323540027,
|
|
"grad_norm": 6.776138063327284,
|
|
"learning_rate": 2.8388291325388024e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"epoch": 0.5118024976539378,
|
|
"grad_norm": 9.638830345053352,
|
|
"learning_rate": 2.832586374837215e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1917,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"epoch": 0.5125243629538728,
|
|
"grad_norm": 10.274548246717263,
|
|
"learning_rate": 2.826341505203047e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"epoch": 0.5132462282538078,
|
|
"grad_norm": 10.65030499452255,
|
|
"learning_rate": 2.8200945632913917e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"epoch": 0.5139680935537428,
|
|
"grad_norm": 8.65595924084498,
|
|
"learning_rate": 2.8138455887705017e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1748,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"epoch": 0.5146899588536779,
|
|
"grad_norm": 7.416014752541617,
|
|
"learning_rate": 2.807594621321536e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"epoch": 0.515411824153613,
|
|
"grad_norm": 9.986965415691905,
|
|
"learning_rate": 2.801341700638307e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"epoch": 0.516133689453548,
|
|
"grad_norm": 8.63015173531394,
|
|
"learning_rate": 2.7950868664270355e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"epoch": 0.516855554753483,
|
|
"grad_norm": 8.92932402159572,
|
|
"learning_rate": 2.788830158406088e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"epoch": 0.517577420053418,
|
|
"grad_norm": 9.18548337260447,
|
|
"learning_rate": 2.782571616305731e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1761,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"epoch": 0.518299285353353,
|
|
"grad_norm": 7.559246854549426,
|
|
"learning_rate": 2.7763112798678787e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"epoch": 0.5190211506532881,
|
|
"grad_norm": 8.366664721309567,
|
|
"learning_rate": 2.77004918884584e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"epoch": 0.5197430159532231,
|
|
"grad_norm": 11.737242043765724,
|
|
"learning_rate": 2.763785383004063e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.5204648812531582,
|
|
"grad_norm": 7.926610565257705,
|
|
"learning_rate": 2.7575199021178855e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1644,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"epoch": 0.5211867465530932,
|
|
"grad_norm": 11.714571880510713,
|
|
"learning_rate": 2.751252785973284e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"epoch": 0.5219086118530282,
|
|
"grad_norm": 8.471661619696453,
|
|
"learning_rate": 2.744984074366617e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"epoch": 0.5226304771529633,
|
|
"grad_norm": 9.494843803969578,
|
|
"learning_rate": 2.738713807104375e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"epoch": 0.5233523424528983,
|
|
"grad_norm": 7.787106376583822,
|
|
"learning_rate": 2.732442024002926e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1461,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"epoch": 0.5240742077528333,
|
|
"grad_norm": 6.681800096835566,
|
|
"learning_rate": 2.726168764888264e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1681,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"epoch": 0.5247960730527683,
|
|
"grad_norm": 9.923678846098278,
|
|
"learning_rate": 2.7198940695957573e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"epoch": 0.5255179383527034,
|
|
"grad_norm": 9.08851776389925,
|
|
"learning_rate": 2.713617977969892e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2118,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"epoch": 0.5262398036526384,
|
|
"grad_norm": 9.6683925726083,
|
|
"learning_rate": 2.707340529864022e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"epoch": 0.5269616689525735,
|
|
"grad_norm": 10.896898496114463,
|
|
"learning_rate": 2.7010617651401133e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"epoch": 0.5276835342525085,
|
|
"grad_norm": 10.726650561848246,
|
|
"learning_rate": 2.694781723668495e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"epoch": 0.5284053995524435,
|
|
"grad_norm": 10.009720787997963,
|
|
"learning_rate": 2.6885004453276014e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"epoch": 0.5291272648523786,
|
|
"grad_norm": 10.393937281174756,
|
|
"learning_rate": 2.682217970003724e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1809,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"epoch": 0.5298491301523136,
|
|
"grad_norm": 6.564234569234908,
|
|
"learning_rate": 2.6759343375907497e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1776,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"epoch": 0.5305709954522486,
|
|
"grad_norm": 8.875705075362786,
|
|
"learning_rate": 2.669649587989918e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"epoch": 0.5312928607521836,
|
|
"grad_norm": 8.149716947695198,
|
|
"learning_rate": 2.6633637611095615e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"epoch": 0.5320147260521186,
|
|
"grad_norm": 9.117623642609074,
|
|
"learning_rate": 2.6570768968648515e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"epoch": 0.5327365913520538,
|
|
"grad_norm": 7.7833308392028115,
|
|
"learning_rate": 2.6507890351775485e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"epoch": 0.5334584566519888,
|
|
"grad_norm": 11.143661735085152,
|
|
"learning_rate": 2.644500215975747e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.177,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"epoch": 0.5341803219519238,
|
|
"grad_norm": 6.014943903391089,
|
|
"learning_rate": 2.63821047919362e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1905,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.5349021872518588,
|
|
"grad_norm": 9.506926726417404,
|
|
"learning_rate": 2.631919864771168e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"epoch": 0.5356240525517938,
|
|
"grad_norm": 8.989178126574519,
|
|
"learning_rate": 2.6256284126539657e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"epoch": 0.5363459178517289,
|
|
"grad_norm": 8.096664424589033,
|
|
"learning_rate": 2.619336162792905e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"epoch": 0.5370677831516639,
|
|
"grad_norm": 16.841361704172662,
|
|
"learning_rate": 2.6130431551439456e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"epoch": 0.5377896484515989,
|
|
"grad_norm": 10.149973787308852,
|
|
"learning_rate": 2.606749429667859e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1571,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"epoch": 0.5385115137515339,
|
|
"grad_norm": 11.046710856292618,
|
|
"learning_rate": 2.6004550263299724e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1731,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"epoch": 0.539233379051469,
|
|
"grad_norm": 9.061807143540431,
|
|
"learning_rate": 2.5941599850999194e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1776,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"epoch": 0.5399552443514041,
|
|
"grad_norm": 10.136512372235272,
|
|
"learning_rate": 2.5878643459513857e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"epoch": 0.5406771096513391,
|
|
"grad_norm": 9.091121034585713,
|
|
"learning_rate": 2.58156814886185e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"epoch": 0.5413989749512741,
|
|
"grad_norm": 11.023486939102211,
|
|
"learning_rate": 2.575271433812338e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 0.5421208402512091,
|
|
"grad_norm": 12.378483126658342,
|
|
"learning_rate": 2.5689742407871615e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"epoch": 0.5428427055511441,
|
|
"grad_norm": 9.630319451714357,
|
|
"learning_rate": 2.562676609773669e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"epoch": 0.5435645708510792,
|
|
"grad_norm": 8.457889597977909,
|
|
"learning_rate": 2.556378580761989e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1797,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"epoch": 0.5442864361510142,
|
|
"grad_norm": 8.046649935179103,
|
|
"learning_rate": 2.5500801937447794e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1379,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"epoch": 0.5450083014509493,
|
|
"grad_norm": 9.046127207562416,
|
|
"learning_rate": 2.5437814887169684e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"epoch": 0.5457301667508843,
|
|
"grad_norm": 9.329772174624635,
|
|
"learning_rate": 2.5374825056755073e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"epoch": 0.5464520320508193,
|
|
"grad_norm": 9.9179656907273,
|
|
"learning_rate": 2.531183284619109e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"epoch": 0.5471738973507544,
|
|
"grad_norm": 9.557689966293122,
|
|
"learning_rate": 2.524883865548e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1632,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"epoch": 0.5478957626506894,
|
|
"grad_norm": 9.086340027521503,
|
|
"learning_rate": 2.5185842884636633e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"epoch": 0.5486176279506244,
|
|
"grad_norm": 8.606477368401597,
|
|
"learning_rate": 2.5122845933685873e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.5493394932505594,
|
|
"grad_norm": 9.684120993406607,
|
|
"learning_rate": 2.505984820266007e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"epoch": 0.5500613585504944,
|
|
"grad_norm": 11.673738089201613,
|
|
"learning_rate": 2.4996850091596535e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"epoch": 0.5507832238504295,
|
|
"grad_norm": 9.43384042742059,
|
|
"learning_rate": 2.493385200053502e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"epoch": 0.5515050891503646,
|
|
"grad_norm": 7.477187732924252,
|
|
"learning_rate": 2.48708543295151e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1651,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"epoch": 0.5522269544502996,
|
|
"grad_norm": 7.419622643126169,
|
|
"learning_rate": 2.480785747857372e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"epoch": 0.5529488197502346,
|
|
"grad_norm": 7.6418997353667955,
|
|
"learning_rate": 2.474486184774262e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"epoch": 0.5536706850501696,
|
|
"grad_norm": 11.143956805660984,
|
|
"learning_rate": 2.468186783704575e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"epoch": 0.5543925503501047,
|
|
"grad_norm": 12.296442116097905,
|
|
"learning_rate": 2.461887584649684e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"epoch": 0.5551144156500397,
|
|
"grad_norm": 13.050278426508344,
|
|
"learning_rate": 2.4555886276096713e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1799,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"epoch": 0.5558362809499747,
|
|
"grad_norm": 7.331592070155222,
|
|
"learning_rate": 2.4492899525830896e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"epoch": 0.5565581462499097,
|
|
"grad_norm": 10.969811602721066,
|
|
"learning_rate": 2.4429915995666967e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"epoch": 0.5572800115498447,
|
|
"grad_norm": 6.714447148185967,
|
|
"learning_rate": 2.4366936085552055e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1493,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"epoch": 0.5580018768497799,
|
|
"grad_norm": 6.283567486920552,
|
|
"learning_rate": 2.430396019541032e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"epoch": 0.5587237421497149,
|
|
"grad_norm": 9.641423565186832,
|
|
"learning_rate": 2.42409887251404e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"epoch": 0.5594456074496499,
|
|
"grad_norm": 9.82613619768625,
|
|
"learning_rate": 2.4178022074612834e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"epoch": 0.5601674727495849,
|
|
"grad_norm": 8.116478311651692,
|
|
"learning_rate": 2.4115060643667596e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"epoch": 0.56088933804952,
|
|
"grad_norm": 9.994907736669813,
|
|
"learning_rate": 2.4052104832111477e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"epoch": 0.561611203349455,
|
|
"grad_norm": 8.76711793301904,
|
|
"learning_rate": 2.39891550397156e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"epoch": 0.56233306864939,
|
|
"grad_norm": 8.295373017899776,
|
|
"learning_rate": 2.392621166621289e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1816,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"epoch": 0.563054933949325,
|
|
"grad_norm": 7.740744626757316,
|
|
"learning_rate": 2.386327511129548e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.56377679924926,
|
|
"grad_norm": 10.604529688792997,
|
|
"learning_rate": 2.380034577461221e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1593,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"epoch": 0.5644986645491952,
|
|
"grad_norm": 6.778093404198004,
|
|
"learning_rate": 2.373742405576611e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"epoch": 0.5652205298491302,
|
|
"grad_norm": 10.12858521011711,
|
|
"learning_rate": 2.3674510354311787e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1778,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7830
|
|
},
|
|
{
|
|
"epoch": 0.5659423951490652,
|
|
"grad_norm": 8.064195843692932,
|
|
"learning_rate": 2.3611605069752988e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"epoch": 0.5666642604490002,
|
|
"grad_norm": 9.775841021106938,
|
|
"learning_rate": 2.3548708601539982e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7850
|
|
},
|
|
{
|
|
"epoch": 0.5673861257489352,
|
|
"grad_norm": 10.098938805871972,
|
|
"learning_rate": 2.3485821349067062e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"epoch": 0.5681079910488703,
|
|
"grad_norm": 8.30412143652236,
|
|
"learning_rate": 2.3422943711670015e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7870
|
|
},
|
|
{
|
|
"epoch": 0.5688298563488053,
|
|
"grad_norm": 6.817526291974242,
|
|
"learning_rate": 2.3360076088623544e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"epoch": 0.5695517216487404,
|
|
"grad_norm": 10.06504278640758,
|
|
"learning_rate": 2.3297218879138775e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7890
|
|
},
|
|
{
|
|
"epoch": 0.5702735869486754,
|
|
"grad_norm": 7.820665146752294,
|
|
"learning_rate": 2.3234372482360726e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"epoch": 0.5709954522486104,
|
|
"grad_norm": 11.261516761461985,
|
|
"learning_rate": 2.3171537297365718e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1524,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7910
|
|
},
|
|
{
|
|
"epoch": 0.5717173175485455,
|
|
"grad_norm": 12.326978372880708,
|
|
"learning_rate": 2.3108713723158908e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1601,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"epoch": 0.5724391828484805,
|
|
"grad_norm": 12.561004648741765,
|
|
"learning_rate": 2.304590215867169e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7930
|
|
},
|
|
{
|
|
"epoch": 0.5731610481484155,
|
|
"grad_norm": 7.388599673863509,
|
|
"learning_rate": 2.298310300275924e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1454,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"epoch": 0.5738829134483505,
|
|
"grad_norm": 6.502079922036917,
|
|
"learning_rate": 2.2920316654197912e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7950
|
|
},
|
|
{
|
|
"epoch": 0.5746047787482855,
|
|
"grad_norm": 8.707306377676787,
|
|
"learning_rate": 2.2857543511682721e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"epoch": 0.5753266440482206,
|
|
"grad_norm": 9.94916607775227,
|
|
"learning_rate": 2.279478397382486e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7970
|
|
},
|
|
{
|
|
"epoch": 0.5760485093481557,
|
|
"grad_norm": 6.43085308116789,
|
|
"learning_rate": 2.273203843914911e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"epoch": 0.5767703746480907,
|
|
"grad_norm": 9.384961354414472,
|
|
"learning_rate": 2.266930730609132e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1473,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7990
|
|
},
|
|
{
|
|
"epoch": 0.5774922399480257,
|
|
"grad_norm": 7.869464671685872,
|
|
"learning_rate": 2.260659097299591e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 0.5782141052479607,
|
|
"grad_norm": 6.036220985959268,
|
|
"learning_rate": 2.2543889838113297e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1523,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 8010
|
|
},
|
|
{
|
|
"epoch": 0.5789359705478958,
|
|
"grad_norm": 9.091611380745027,
|
|
"learning_rate": 2.2481204299597422e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1708,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"epoch": 0.5796578358478308,
|
|
"grad_norm": 8.969271615418297,
|
|
"learning_rate": 2.2418534755503169e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.154,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8030
|
|
},
|
|
{
|
|
"epoch": 0.5803797011477658,
|
|
"grad_norm": 8.869611211845172,
|
|
"learning_rate": 2.235588160378384e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1892,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"epoch": 0.5811015664477008,
|
|
"grad_norm": 8.585563297437242,
|
|
"learning_rate": 2.229324524228868e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1972,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8050
|
|
},
|
|
{
|
|
"epoch": 0.5818234317476358,
|
|
"grad_norm": 6.361935798258764,
|
|
"learning_rate": 2.2230626068760297e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"epoch": 0.582545297047571,
|
|
"grad_norm": 7.123503913889777,
|
|
"learning_rate": 2.216802448083214e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1524,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8070
|
|
},
|
|
{
|
|
"epoch": 0.583267162347506,
|
|
"grad_norm": 5.3274420243546645,
|
|
"learning_rate": 2.2105440876026026e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"epoch": 0.583989027647441,
|
|
"grad_norm": 8.126969483825905,
|
|
"learning_rate": 2.2042875651749548e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1605,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8090
|
|
},
|
|
{
|
|
"epoch": 0.584710892947376,
|
|
"grad_norm": 6.82538345344394,
|
|
"learning_rate": 2.1980329205293585e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1633,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"epoch": 0.585432758247311,
|
|
"grad_norm": 12.200961629937073,
|
|
"learning_rate": 2.1917801933829807e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8110
|
|
},
|
|
{
|
|
"epoch": 0.5861546235472461,
|
|
"grad_norm": 7.906908172237448,
|
|
"learning_rate": 2.1855294234408068e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"epoch": 0.5868764888471811,
|
|
"grad_norm": 10.313898842262107,
|
|
"learning_rate": 2.1792806503953986e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8130
|
|
},
|
|
{
|
|
"epoch": 0.5875983541471161,
|
|
"grad_norm": 9.428346584688365,
|
|
"learning_rate": 2.1730339139266368e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"epoch": 0.5883202194470512,
|
|
"grad_norm": 9.348567817555846,
|
|
"learning_rate": 2.1667892537014664e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8150
|
|
},
|
|
{
|
|
"epoch": 0.5890420847469862,
|
|
"grad_norm": 9.578462386462538,
|
|
"learning_rate": 2.1605467093736534e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1611,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"epoch": 0.5897639500469213,
|
|
"grad_norm": 7.084598817360508,
|
|
"learning_rate": 2.1543063205835232e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1716,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8170
|
|
},
|
|
{
|
|
"epoch": 0.5904858153468563,
|
|
"grad_norm": 10.096964364918204,
|
|
"learning_rate": 2.1480681269577156e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"epoch": 0.5912076806467913,
|
|
"grad_norm": 13.036565026698682,
|
|
"learning_rate": 2.141832168108932e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8190
|
|
},
|
|
{
|
|
"epoch": 0.5919295459467263,
|
|
"grad_norm": 10.503733261465367,
|
|
"learning_rate": 2.1355984836356805e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"epoch": 0.5926514112466613,
|
|
"grad_norm": 8.837085032744609,
|
|
"learning_rate": 2.1293671131220277e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1517,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8210
|
|
},
|
|
{
|
|
"epoch": 0.5933732765465964,
|
|
"grad_norm": 13.492509568178333,
|
|
"learning_rate": 2.123138096137349e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1718,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"epoch": 0.5940951418465314,
|
|
"grad_norm": 4.422649076334775,
|
|
"learning_rate": 2.1169114722360708e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1322,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.5,
|
|
"step": 8230
|
|
},
|
|
{
|
|
"epoch": 0.5948170071464665,
|
|
"grad_norm": 8.207509821314797,
|
|
"learning_rate": 2.1106872809574266e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"epoch": 0.5955388724464015,
|
|
"grad_norm": 11.51646939268645,
|
|
"learning_rate": 2.1044655618251994e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8250
|
|
},
|
|
{
|
|
"epoch": 0.5962607377463366,
|
|
"grad_norm": 8.81738149975298,
|
|
"learning_rate": 2.098246354347477e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"epoch": 0.5969826030462716,
|
|
"grad_norm": 9.880728738377933,
|
|
"learning_rate": 2.0920296980163975e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8270
|
|
},
|
|
{
|
|
"epoch": 0.5977044683462066,
|
|
"grad_norm": 12.130445254705615,
|
|
"learning_rate": 2.0858156323078986e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"epoch": 0.5984263336461416,
|
|
"grad_norm": 9.18646238410699,
|
|
"learning_rate": 2.0796041966814658e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8290
|
|
},
|
|
{
|
|
"epoch": 0.5991481989460766,
|
|
"grad_norm": 7.4415035510148595,
|
|
"learning_rate": 2.0733954305798884e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1668,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"epoch": 0.5998700642460117,
|
|
"grad_norm": 9.921182858419193,
|
|
"learning_rate": 2.0671893734289984e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8310
|
|
},
|
|
{
|
|
"epoch": 0.6005919295459468,
|
|
"grad_norm": 6.738345944941375,
|
|
"learning_rate": 2.0609860646374305e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1682,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"epoch": 0.6013137948458818,
|
|
"grad_norm": 9.581111671762294,
|
|
"learning_rate": 2.054785543596363e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.174,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8330
|
|
},
|
|
{
|
|
"epoch": 0.6020356601458168,
|
|
"grad_norm": 9.266451111611559,
|
|
"learning_rate": 2.0485878496792758e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1594,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"epoch": 0.6027575254457518,
|
|
"grad_norm": 6.924514229924403,
|
|
"learning_rate": 2.042393022241695e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1331,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 8350
|
|
},
|
|
{
|
|
"epoch": 0.6034793907456869,
|
|
"grad_norm": 11.878033012527993,
|
|
"learning_rate": 2.0362011006209429e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"epoch": 0.6042012560456219,
|
|
"grad_norm": 6.456356558581018,
|
|
"learning_rate": 2.0300121241358925e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1412,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8370
|
|
},
|
|
{
|
|
"epoch": 0.6049231213455569,
|
|
"grad_norm": 6.2766000485447595,
|
|
"learning_rate": 2.023826132086714e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"epoch": 0.6056449866454919,
|
|
"grad_norm": 10.010107774455438,
|
|
"learning_rate": 2.0176431637546243e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8390
|
|
},
|
|
{
|
|
"epoch": 0.6063668519454269,
|
|
"grad_norm": 6.93155516769275,
|
|
"learning_rate": 2.0114632584016444e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1442,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"epoch": 0.6070887172453621,
|
|
"grad_norm": 9.44671796286649,
|
|
"learning_rate": 2.0052864552703392e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8410
|
|
},
|
|
{
|
|
"epoch": 0.6078105825452971,
|
|
"grad_norm": 10.256129872630872,
|
|
"learning_rate": 1.9991127935835796e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"epoch": 0.6085324478452321,
|
|
"grad_norm": 10.112281066079031,
|
|
"learning_rate": 1.9929423125442866e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8430
|
|
},
|
|
{
|
|
"epoch": 0.6092543131451671,
|
|
"grad_norm": 5.989996318215951,
|
|
"learning_rate": 1.9867750513351813e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1405,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"epoch": 0.6099761784451021,
|
|
"grad_norm": 7.127345044430176,
|
|
"learning_rate": 1.9806110491185434e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1401,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8450
|
|
},
|
|
{
|
|
"epoch": 0.6106980437450372,
|
|
"grad_norm": 8.882033575289205,
|
|
"learning_rate": 1.9744503450359555e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"epoch": 0.6114199090449722,
|
|
"grad_norm": 11.636675089992194,
|
|
"learning_rate": 1.968292978208055e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8470
|
|
},
|
|
{
|
|
"epoch": 0.6121417743449072,
|
|
"grad_norm": 4.708987187458577,
|
|
"learning_rate": 1.9621389877342926e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"epoch": 0.6128636396448423,
|
|
"grad_norm": 8.691036530455966,
|
|
"learning_rate": 1.9559884126926742e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8490
|
|
},
|
|
{
|
|
"epoch": 0.6135855049447773,
|
|
"grad_norm": 7.033771928824113,
|
|
"learning_rate": 1.9498412921395196e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"epoch": 0.6143073702447124,
|
|
"grad_norm": 6.420805831736498,
|
|
"learning_rate": 1.9436976651092142e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1517,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8510
|
|
},
|
|
{
|
|
"epoch": 0.6150292355446474,
|
|
"grad_norm": 9.804155572476585,
|
|
"learning_rate": 1.9375575706139557e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"epoch": 0.6157511008445824,
|
|
"grad_norm": 8.402725478167426,
|
|
"learning_rate": 1.931421047643513e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8530
|
|
},
|
|
{
|
|
"epoch": 0.6164729661445174,
|
|
"grad_norm": 7.928649293353565,
|
|
"learning_rate": 1.9252881351649764e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"epoch": 0.6171948314444524,
|
|
"grad_norm": 10.584642277815288,
|
|
"learning_rate": 1.919158872122505e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1709,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8550
|
|
},
|
|
{
|
|
"epoch": 0.6179166967443875,
|
|
"grad_norm": 5.528300128432007,
|
|
"learning_rate": 1.9130332974370888e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"epoch": 0.6186385620443225,
|
|
"grad_norm": 8.381790511594774,
|
|
"learning_rate": 1.9069114500062944e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8570
|
|
},
|
|
{
|
|
"epoch": 0.6193604273442576,
|
|
"grad_norm": 7.672005385585843,
|
|
"learning_rate": 1.9007933687040192e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1556,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"epoch": 0.6200822926441926,
|
|
"grad_norm": 7.6497315518223505,
|
|
"learning_rate": 1.8946790923802491e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8590
|
|
},
|
|
{
|
|
"epoch": 0.6208041579441276,
|
|
"grad_norm": 8.83669969757212,
|
|
"learning_rate": 1.8885686598608044e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"epoch": 0.6215260232440627,
|
|
"grad_norm": 11.726451803340803,
|
|
"learning_rate": 1.8824621099470986e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8610
|
|
},
|
|
{
|
|
"epoch": 0.6222478885439977,
|
|
"grad_norm": 6.926635938541544,
|
|
"learning_rate": 1.8763594814158926e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"epoch": 0.6229697538439327,
|
|
"grad_norm": 29.047689387473163,
|
|
"learning_rate": 1.8702608130190427e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8630
|
|
},
|
|
{
|
|
"epoch": 0.6236916191438677,
|
|
"grad_norm": 12.046850711595459,
|
|
"learning_rate": 1.8641661434832615e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"epoch": 0.6244134844438027,
|
|
"grad_norm": 13.389050315360532,
|
|
"learning_rate": 1.858075511509865e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1718,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8650
|
|
},
|
|
{
|
|
"epoch": 0.6251353497437379,
|
|
"grad_norm": 8.48405716596924,
|
|
"learning_rate": 1.8519889557745355e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"epoch": 0.6258572150436729,
|
|
"grad_norm": 7.718084904787592,
|
|
"learning_rate": 1.8459065149270675e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.204,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8670
|
|
},
|
|
{
|
|
"epoch": 0.6265790803436079,
|
|
"grad_norm": 8.645603679282008,
|
|
"learning_rate": 1.8398282275911265e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"epoch": 0.6273009456435429,
|
|
"grad_norm": 9.649146909089938,
|
|
"learning_rate": 1.833754132364003e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8690
|
|
},
|
|
{
|
|
"epoch": 0.6280228109434779,
|
|
"grad_norm": 10.143836034702232,
|
|
"learning_rate": 1.8276842678163694e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"epoch": 0.628744676243413,
|
|
"grad_norm": 6.224441882099418,
|
|
"learning_rate": 1.82161867249203e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 8710
|
|
},
|
|
{
|
|
"epoch": 0.629466541543348,
|
|
"grad_norm": 10.818946600627243,
|
|
"learning_rate": 1.8155573849076829e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"epoch": 0.630188406843283,
|
|
"grad_norm": 8.600830363109214,
|
|
"learning_rate": 1.8095004435526673e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8730
|
|
},
|
|
{
|
|
"epoch": 0.630910272143218,
|
|
"grad_norm": 7.278694825397051,
|
|
"learning_rate": 1.8034478868887293e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"epoch": 0.631632137443153,
|
|
"grad_norm": 10.061499794489972,
|
|
"learning_rate": 1.7973997533497684e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1721,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8750
|
|
},
|
|
{
|
|
"epoch": 0.6323540027430882,
|
|
"grad_norm": 12.377386224781029,
|
|
"learning_rate": 1.791356081341597e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.152,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"epoch": 0.6330758680430232,
|
|
"grad_norm": 7.666515466883803,
|
|
"learning_rate": 1.7853169092416992e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8770
|
|
},
|
|
{
|
|
"epoch": 0.6337977333429582,
|
|
"grad_norm": 9.2022093037141,
|
|
"learning_rate": 1.779282275398983e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"epoch": 0.6345195986428932,
|
|
"grad_norm": 11.875630627682433,
|
|
"learning_rate": 1.773252218133537e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 8790
|
|
},
|
|
{
|
|
"epoch": 0.6352414639428283,
|
|
"grad_norm": 12.286853044802243,
|
|
"learning_rate": 1.7672267757363917e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"epoch": 0.6359633292427633,
|
|
"grad_norm": 7.782715805579312,
|
|
"learning_rate": 1.7612059864692696e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1434,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8810
|
|
},
|
|
{
|
|
"epoch": 0.6366851945426983,
|
|
"grad_norm": 10.338787396379217,
|
|
"learning_rate": 1.7551898885643478e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1478,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"epoch": 0.6374070598426334,
|
|
"grad_norm": 8.641921016163051,
|
|
"learning_rate": 1.749178520224014e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1519,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8830
|
|
},
|
|
{
|
|
"epoch": 0.6381289251425684,
|
|
"grad_norm": 9.128585650717545,
|
|
"learning_rate": 1.7431719196206186e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"epoch": 0.6388507904425035,
|
|
"grad_norm": 11.694392580125832,
|
|
"learning_rate": 1.7371701248962426e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1925,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8850
|
|
},
|
|
{
|
|
"epoch": 0.6395726557424385,
|
|
"grad_norm": 8.29325313521127,
|
|
"learning_rate": 1.7311731741624458e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"epoch": 0.6402945210423735,
|
|
"grad_norm": 8.814547322060793,
|
|
"learning_rate": 1.7251811055000283e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1451,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8870
|
|
},
|
|
{
|
|
"epoch": 0.6410163863423085,
|
|
"grad_norm": 9.320265993143382,
|
|
"learning_rate": 1.7191939569587923e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.13,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"epoch": 0.6417382516422435,
|
|
"grad_norm": 10.838220312842498,
|
|
"learning_rate": 1.7132117665572932e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8890
|
|
},
|
|
{
|
|
"epoch": 0.6424601169421786,
|
|
"grad_norm": 6.55136471958019,
|
|
"learning_rate": 1.7072345722826035e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1315,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"epoch": 0.6431819822421136,
|
|
"grad_norm": 8.66688500340224,
|
|
"learning_rate": 1.701262412090072e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8910
|
|
},
|
|
{
|
|
"epoch": 0.6439038475420487,
|
|
"grad_norm": 9.629379415169119,
|
|
"learning_rate": 1.6952953239030779e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1416,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"epoch": 0.6446257128419837,
|
|
"grad_norm": 12.098110662139495,
|
|
"learning_rate": 1.6893333456127946e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8930
|
|
},
|
|
{
|
|
"epoch": 0.6453475781419187,
|
|
"grad_norm": 8.256462573405791,
|
|
"learning_rate": 1.6833765150779484e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1609,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"epoch": 0.6460694434418538,
|
|
"grad_norm": 11.239144635721424,
|
|
"learning_rate": 1.677424870124575e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8950
|
|
},
|
|
{
|
|
"epoch": 0.6467913087417888,
|
|
"grad_norm": 8.418303487457777,
|
|
"learning_rate": 1.671478448545784e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.149,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"epoch": 0.6475131740417238,
|
|
"grad_norm": 9.492259339751383,
|
|
"learning_rate": 1.665537288101514e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8970
|
|
},
|
|
{
|
|
"epoch": 0.6482350393416588,
|
|
"grad_norm": 7.953989568068451,
|
|
"learning_rate": 1.6596014265182957e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1683,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"epoch": 0.6489569046415938,
|
|
"grad_norm": 7.299757937508117,
|
|
"learning_rate": 1.6536709014890147e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1577,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8990
|
|
},
|
|
{
|
|
"epoch": 0.649678769941529,
|
|
"grad_norm": 9.426751785884438,
|
|
"learning_rate": 1.6477457506726655e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 0.650400635241464,
|
|
"grad_norm": 10.774886673375532,
|
|
"learning_rate": 1.641826011694119e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9010
|
|
},
|
|
{
|
|
"epoch": 0.651122500541399,
|
|
"grad_norm": 9.624000604333325,
|
|
"learning_rate": 1.6359117221438784e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"epoch": 0.651844365841334,
|
|
"grad_norm": 9.17261641216877,
|
|
"learning_rate": 1.6300029195778453e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9030
|
|
},
|
|
{
|
|
"epoch": 0.652566231141269,
|
|
"grad_norm": 10.974103824549372,
|
|
"learning_rate": 1.624099641517078e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"epoch": 0.6532880964412041,
|
|
"grad_norm": 6.917901057511971,
|
|
"learning_rate": 1.6182019254475514e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1515,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9050
|
|
},
|
|
{
|
|
"epoch": 0.6540099617411391,
|
|
"grad_norm": 10.04014608485764,
|
|
"learning_rate": 1.6123098088199267e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1476,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"epoch": 0.6547318270410741,
|
|
"grad_norm": 11.307618522526226,
|
|
"learning_rate": 1.6064233290493048e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1568,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9070
|
|
},
|
|
{
|
|
"epoch": 0.6554536923410091,
|
|
"grad_norm": 11.567312788735324,
|
|
"learning_rate": 1.600542523514992e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"epoch": 0.6561755576409442,
|
|
"grad_norm": 6.469353343465349,
|
|
"learning_rate": 1.5946674295602668e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1644,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9090
|
|
},
|
|
{
|
|
"epoch": 0.6568974229408793,
|
|
"grad_norm": 9.939174768893757,
|
|
"learning_rate": 1.5887980844921338e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1646,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"epoch": 0.6576192882408143,
|
|
"grad_norm": 10.710746815227962,
|
|
"learning_rate": 1.5829345255810966e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9110
|
|
},
|
|
{
|
|
"epoch": 0.6583411535407493,
|
|
"grad_norm": 9.589016893008033,
|
|
"learning_rate": 1.5770767900609144e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1403,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"epoch": 0.6590630188406843,
|
|
"grad_norm": 10.535090311502815,
|
|
"learning_rate": 1.5712249151283674e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9130
|
|
},
|
|
{
|
|
"epoch": 0.6597848841406193,
|
|
"grad_norm": 11.07484592479438,
|
|
"learning_rate": 1.565378937943022e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"epoch": 0.6605067494405544,
|
|
"grad_norm": 8.804459530019093,
|
|
"learning_rate": 1.559538895626994e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9150
|
|
},
|
|
{
|
|
"epoch": 0.6612286147404894,
|
|
"grad_norm": 9.869851207701721,
|
|
"learning_rate": 1.5537048252647102e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"epoch": 0.6619504800404244,
|
|
"grad_norm": 7.227577491074705,
|
|
"learning_rate": 1.547876763902679e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1604,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9170
|
|
},
|
|
{
|
|
"epoch": 0.6626723453403595,
|
|
"grad_norm": 8.512681174359171,
|
|
"learning_rate": 1.5420547485492483e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"epoch": 0.6633942106402945,
|
|
"grad_norm": 6.940912396524942,
|
|
"learning_rate": 1.5362388161743743e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9190
|
|
},
|
|
{
|
|
"epoch": 0.6641160759402296,
|
|
"grad_norm": 7.482638973278819,
|
|
"learning_rate": 1.5304290037093887e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"epoch": 0.6648379412401646,
|
|
"grad_norm": 9.239067837458254,
|
|
"learning_rate": 1.524625348046758e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1324,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9210
|
|
},
|
|
{
|
|
"epoch": 0.6655598065400996,
|
|
"grad_norm": 10.312943276531238,
|
|
"learning_rate": 1.5188278860398543e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1548,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"epoch": 0.6662816718400346,
|
|
"grad_norm": 12.377500015344213,
|
|
"learning_rate": 1.5130366545027215e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9230
|
|
},
|
|
{
|
|
"epoch": 0.6670035371399696,
|
|
"grad_norm": 11.437994745400522,
|
|
"learning_rate": 1.507251690209837e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"epoch": 0.6677254024399047,
|
|
"grad_norm": 9.227052828667116,
|
|
"learning_rate": 1.5014730298958834e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1733,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9250
|
|
},
|
|
{
|
|
"epoch": 0.6684472677398398,
|
|
"grad_norm": 11.34701390652657,
|
|
"learning_rate": 1.4957007102555101e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"epoch": 0.6691691330397748,
|
|
"grad_norm": 10.840533844995983,
|
|
"learning_rate": 1.489934767943106e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9270
|
|
},
|
|
{
|
|
"epoch": 0.6698909983397098,
|
|
"grad_norm": 5.005020117833695,
|
|
"learning_rate": 1.484175239572563e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"epoch": 0.6706128636396449,
|
|
"grad_norm": 9.438942465976393,
|
|
"learning_rate": 1.4784221617170427e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1544,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9290
|
|
},
|
|
{
|
|
"epoch": 0.6713347289395799,
|
|
"grad_norm": 8.775509056317704,
|
|
"learning_rate": 1.4726755709087467e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1568,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"epoch": 0.6720565942395149,
|
|
"grad_norm": 12.815148122117906,
|
|
"learning_rate": 1.466935503638686e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1535,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9310
|
|
},
|
|
{
|
|
"epoch": 0.6727784595394499,
|
|
"grad_norm": 6.322537492099079,
|
|
"learning_rate": 1.4612019963564425e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"epoch": 0.6735003248393849,
|
|
"grad_norm": 8.684754690608713,
|
|
"learning_rate": 1.4554750854699454e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 9330
|
|
},
|
|
{
|
|
"epoch": 0.67422219013932,
|
|
"grad_norm": 7.495450070953094,
|
|
"learning_rate": 1.4497548073452353e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"epoch": 0.6749440554392551,
|
|
"grad_norm": 9.818417235804333,
|
|
"learning_rate": 1.444041198306235e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9350
|
|
},
|
|
{
|
|
"epoch": 0.6756659207391901,
|
|
"grad_norm": 10.0125757389507,
|
|
"learning_rate": 1.4383342946345178e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"epoch": 0.6763877860391251,
|
|
"grad_norm": 10.056618279215506,
|
|
"learning_rate": 1.432634132569079e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9370
|
|
},
|
|
{
|
|
"epoch": 0.6771096513390601,
|
|
"grad_norm": 12.261509335918284,
|
|
"learning_rate": 1.4269407483061025e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"epoch": 0.6778315166389952,
|
|
"grad_norm": 9.583901573024836,
|
|
"learning_rate": 1.4212541779987358e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1808,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9390
|
|
},
|
|
{
|
|
"epoch": 0.6785533819389302,
|
|
"grad_norm": 7.602147425073978,
|
|
"learning_rate": 1.415574457756853e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1429,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"epoch": 0.6792752472388652,
|
|
"grad_norm": 5.715088913700748,
|
|
"learning_rate": 1.409901623646837e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9410
|
|
},
|
|
{
|
|
"epoch": 0.6799971125388002,
|
|
"grad_norm": 7.623897333532847,
|
|
"learning_rate": 1.4042357116913365e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"epoch": 0.6807189778387353,
|
|
"grad_norm": 12.781776965070957,
|
|
"learning_rate": 1.3985767578690474e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1763,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9430
|
|
},
|
|
{
|
|
"epoch": 0.6814408431386704,
|
|
"grad_norm": 10.237854404923459,
|
|
"learning_rate": 1.3929247981144845e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"epoch": 0.6821627084386054,
|
|
"grad_norm": 9.894822207763886,
|
|
"learning_rate": 1.387279868317744e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9450
|
|
},
|
|
{
|
|
"epoch": 0.6828845737385404,
|
|
"grad_norm": 9.241271465537533,
|
|
"learning_rate": 1.3816420043242843e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"epoch": 0.6836064390384754,
|
|
"grad_norm": 10.237589705127691,
|
|
"learning_rate": 1.3760112419346986e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9470
|
|
},
|
|
{
|
|
"epoch": 0.6843283043384104,
|
|
"grad_norm": 10.725962046372151,
|
|
"learning_rate": 1.3703876169044802e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"epoch": 0.6850501696383455,
|
|
"grad_norm": 7.6917366331039325,
|
|
"learning_rate": 1.364771164943802e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9490
|
|
},
|
|
{
|
|
"epoch": 0.6857720349382805,
|
|
"grad_norm": 8.728014646166761,
|
|
"learning_rate": 1.3591619217172883e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1455,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"epoch": 0.6864939002382155,
|
|
"grad_norm": 10.677477412212683,
|
|
"learning_rate": 1.3535599228437867e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1626,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9510
|
|
},
|
|
{
|
|
"epoch": 0.6872157655381506,
|
|
"grad_norm": 9.38419202809167,
|
|
"learning_rate": 1.3479652038961432e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"epoch": 0.6879376308380856,
|
|
"grad_norm": 8.766505031785163,
|
|
"learning_rate": 1.3423778004009762e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9530
|
|
},
|
|
{
|
|
"epoch": 0.6886594961380207,
|
|
"grad_norm": 7.070381554572682,
|
|
"learning_rate": 1.3367977478384513e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"epoch": 0.6893813614379557,
|
|
"grad_norm": 13.363793450146552,
|
|
"learning_rate": 1.3312250816420542e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1678,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9550
|
|
},
|
|
{
|
|
"epoch": 0.6901032267378907,
|
|
"grad_norm": 6.54534750827785,
|
|
"learning_rate": 1.3256598371983686e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"epoch": 0.6908250920378257,
|
|
"grad_norm": 8.065890871247143,
|
|
"learning_rate": 1.320102049846849e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1447,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9570
|
|
},
|
|
{
|
|
"epoch": 0.6915469573377607,
|
|
"grad_norm": 8.464118508817007,
|
|
"learning_rate": 1.314551754879595e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1415,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"epoch": 0.6922688226376958,
|
|
"grad_norm": 9.854707772376594,
|
|
"learning_rate": 1.3090089875411337e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.129,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 9590
|
|
},
|
|
{
|
|
"epoch": 0.6929906879376309,
|
|
"grad_norm": 8.672649571433473,
|
|
"learning_rate": 1.30347378302819e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"epoch": 0.6937125532375659,
|
|
"grad_norm": 9.034737955151657,
|
|
"learning_rate": 1.2979461764894608e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9610
|
|
},
|
|
{
|
|
"epoch": 0.6944344185375009,
|
|
"grad_norm": 8.90041609096791,
|
|
"learning_rate": 1.2924262030254022e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"epoch": 0.6951562838374359,
|
|
"grad_norm": 8.665967459471112,
|
|
"learning_rate": 1.2869138976879961e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9630
|
|
},
|
|
{
|
|
"epoch": 0.695878149137371,
|
|
"grad_norm": 7.497535290815869,
|
|
"learning_rate": 1.2814092954805294e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.171,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"epoch": 0.696600014437306,
|
|
"grad_norm": 9.990492914302166,
|
|
"learning_rate": 1.275912431357381e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9650
|
|
},
|
|
{
|
|
"epoch": 0.697321879737241,
|
|
"grad_norm": 7.823463866960965,
|
|
"learning_rate": 1.2704233402237858e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1707,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"epoch": 0.698043745037176,
|
|
"grad_norm": 7.579915303042644,
|
|
"learning_rate": 1.2649420569356217e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9670
|
|
},
|
|
{
|
|
"epoch": 0.698765610337111,
|
|
"grad_norm": 8.800622349499235,
|
|
"learning_rate": 1.2594686162991914e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1402,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"epoch": 0.6994874756370462,
|
|
"grad_norm": 9.327395207500006,
|
|
"learning_rate": 1.2540030530709888e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9690
|
|
},
|
|
{
|
|
"epoch": 0.7002093409369812,
|
|
"grad_norm": 8.376068930149511,
|
|
"learning_rate": 1.248545401957492e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1543,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"epoch": 0.7009312062369162,
|
|
"grad_norm": 7.936678196185944,
|
|
"learning_rate": 1.2430956976149345e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9710
|
|
},
|
|
{
|
|
"epoch": 0.7016530715368512,
|
|
"grad_norm": 12.345046600610592,
|
|
"learning_rate": 1.2376539746490878e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1713,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"epoch": 0.7023749368367862,
|
|
"grad_norm": 9.162811933963393,
|
|
"learning_rate": 1.2322202676150417e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9730
|
|
},
|
|
{
|
|
"epoch": 0.7030968021367213,
|
|
"grad_norm": 10.256111775264358,
|
|
"learning_rate": 1.2267946110169847e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"epoch": 0.7038186674366563,
|
|
"grad_norm": 8.584246074026865,
|
|
"learning_rate": 1.2213770393079846e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9750
|
|
},
|
|
{
|
|
"epoch": 0.7045405327365913,
|
|
"grad_norm": 7.583949089234293,
|
|
"learning_rate": 1.21596758688977e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1512,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"epoch": 0.7052623980365264,
|
|
"grad_norm": 9.44743736005493,
|
|
"learning_rate": 1.2105662881125112e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1652,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9770
|
|
},
|
|
{
|
|
"epoch": 0.7059842633364615,
|
|
"grad_norm": 6.4078324266348705,
|
|
"learning_rate": 1.205173177274604e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"epoch": 0.7067061286363965,
|
|
"grad_norm": 7.907790182080389,
|
|
"learning_rate": 1.1997882886224497e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1424,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9790
|
|
},
|
|
{
|
|
"epoch": 0.7074279939363315,
|
|
"grad_norm": 12.95788224606365,
|
|
"learning_rate": 1.194411656350238e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1558,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"epoch": 0.7081498592362665,
|
|
"grad_norm": 9.166806155356193,
|
|
"learning_rate": 1.1890433145997327e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1731,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9810
|
|
},
|
|
{
|
|
"epoch": 0.7088717245362015,
|
|
"grad_norm": 9.72249270328465,
|
|
"learning_rate": 1.1836832974600483e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"epoch": 0.7095935898361366,
|
|
"grad_norm": 7.965526025357869,
|
|
"learning_rate": 1.1783316389674406e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9830
|
|
},
|
|
{
|
|
"epoch": 0.7103154551360716,
|
|
"grad_norm": 7.929357398986534,
|
|
"learning_rate": 1.1729883731050899e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1315,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"epoch": 0.7110373204360066,
|
|
"grad_norm": 8.634524210955284,
|
|
"learning_rate": 1.1676535338028781e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9850
|
|
},
|
|
{
|
|
"epoch": 0.7117591857359417,
|
|
"grad_norm": 9.065771203848671,
|
|
"learning_rate": 1.1623271549371808e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1781,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"epoch": 0.7124810510358767,
|
|
"grad_norm": 7.949345789146648,
|
|
"learning_rate": 1.1570092703306523e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9870
|
|
},
|
|
{
|
|
"epoch": 0.7132029163358118,
|
|
"grad_norm": 8.559039194521684,
|
|
"learning_rate": 1.1516999137520023e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"epoch": 0.7139247816357468,
|
|
"grad_norm": 8.830447748470409,
|
|
"learning_rate": 1.1463991189157917e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1701,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9890
|
|
},
|
|
{
|
|
"epoch": 0.7146466469356818,
|
|
"grad_norm": 7.4835869108530275,
|
|
"learning_rate": 1.1411069194822124e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"epoch": 0.7153685122356168,
|
|
"grad_norm": 12.056792194890452,
|
|
"learning_rate": 1.1358233490568758e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1486,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9910
|
|
},
|
|
{
|
|
"epoch": 0.7160903775355518,
|
|
"grad_norm": 7.752875524749948,
|
|
"learning_rate": 1.1305484411905986e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1407,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"epoch": 0.716812242835487,
|
|
"grad_norm": 7.773726914737785,
|
|
"learning_rate": 1.12528222937919e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9930
|
|
},
|
|
{
|
|
"epoch": 0.717534108135422,
|
|
"grad_norm": 10.91086916613174,
|
|
"learning_rate": 1.1200247470632392e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1779,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"epoch": 0.718255973435357,
|
|
"grad_norm": 8.136219531246716,
|
|
"learning_rate": 1.1147760276279028e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9950
|
|
},
|
|
{
|
|
"epoch": 0.718977838735292,
|
|
"grad_norm": 12.213084226027048,
|
|
"learning_rate": 1.1095361044026927e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"epoch": 0.719699704035227,
|
|
"grad_norm": 7.447846242333707,
|
|
"learning_rate": 1.1043050106612656e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9970
|
|
},
|
|
{
|
|
"epoch": 0.7204215693351621,
|
|
"grad_norm": 11.497519578105333,
|
|
"learning_rate": 1.0990827796212074e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"epoch": 0.7211434346350971,
|
|
"grad_norm": 9.931460815710798,
|
|
"learning_rate": 1.0938694444438307e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9990
|
|
},
|
|
{
|
|
"epoch": 0.7218652999350321,
|
|
"grad_norm": 9.875757410099292,
|
|
"learning_rate": 1.0886650382339566e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.7218652999350321,
|
|
"eval_logits/chosen": -2.9375,
|
|
"eval_logits/rejected": -2.625,
|
|
"eval_logps/chosen": -772.0,
|
|
"eval_logps/rejected": -1168.0,
|
|
"eval_loss": 0.21195410192012787,
|
|
"eval_rewards/accuracies": 0.9136363863945007,
|
|
"eval_rewards/chosen": -6.0,
|
|
"eval_rewards/margins": 4.1875,
|
|
"eval_rewards/rejected": -10.1875,
|
|
"eval_runtime": 3594.9603,
|
|
"eval_samples_per_second": 27.4,
|
|
"eval_steps_per_second": 0.428,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.7225871652349671,
|
|
"grad_norm": 9.241595039747112,
|
|
"learning_rate": 1.083469594039704e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1528,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10010
|
|
},
|
|
{
|
|
"epoch": 0.7233090305349021,
|
|
"grad_norm": 10.979124343100237,
|
|
"learning_rate": 1.0782831448522886e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10020
|
|
},
|
|
{
|
|
"epoch": 0.7240308958348373,
|
|
"grad_norm": 9.941856346590853,
|
|
"learning_rate": 1.0731057236058047e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10030
|
|
},
|
|
{
|
|
"epoch": 0.7247527611347723,
|
|
"grad_norm": 10.561997010522104,
|
|
"learning_rate": 1.0679373631770162e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1482,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10040
|
|
},
|
|
{
|
|
"epoch": 0.7254746264347073,
|
|
"grad_norm": 8.458140492638268,
|
|
"learning_rate": 1.0627780963851562e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10050
|
|
},
|
|
{
|
|
"epoch": 0.7261964917346423,
|
|
"grad_norm": 7.155521990324457,
|
|
"learning_rate": 1.0576279559917081e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1383,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 10060
|
|
},
|
|
{
|
|
"epoch": 0.7269183570345773,
|
|
"grad_norm": 9.028193158327456,
|
|
"learning_rate": 1.0524869747002041e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1405,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10070
|
|
},
|
|
{
|
|
"epoch": 0.7276402223345124,
|
|
"grad_norm": 7.250067406009502,
|
|
"learning_rate": 1.0473551851560184e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1434,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10080
|
|
},
|
|
{
|
|
"epoch": 0.7283620876344474,
|
|
"grad_norm": 13.036104853884511,
|
|
"learning_rate": 1.0422326199461526e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1419,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10090
|
|
},
|
|
{
|
|
"epoch": 0.7290839529343824,
|
|
"grad_norm": 11.132862883187533,
|
|
"learning_rate": 1.0371193115990362e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 10100
|
|
},
|
|
{
|
|
"epoch": 0.7298058182343174,
|
|
"grad_norm": 11.150633604056837,
|
|
"learning_rate": 1.0320152925843192e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1702,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10110
|
|
},
|
|
{
|
|
"epoch": 0.7305276835342525,
|
|
"grad_norm": 9.295923421645263,
|
|
"learning_rate": 1.02692059531266e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1433,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10120
|
|
},
|
|
{
|
|
"epoch": 0.7312495488341876,
|
|
"grad_norm": 10.553464062235404,
|
|
"learning_rate": 1.0218352521355262e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1472,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10130
|
|
},
|
|
{
|
|
"epoch": 0.7319714141341226,
|
|
"grad_norm": 7.390985068640905,
|
|
"learning_rate": 1.0167592953449858e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1468,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10140
|
|
},
|
|
{
|
|
"epoch": 0.7326932794340576,
|
|
"grad_norm": 9.445556995082786,
|
|
"learning_rate": 1.011692757173504e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10150
|
|
},
|
|
{
|
|
"epoch": 0.7334151447339926,
|
|
"grad_norm": 12.596474294167802,
|
|
"learning_rate": 1.0066356697937362e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1564,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.875,
|
|
"step": 10160
|
|
},
|
|
{
|
|
"epoch": 0.7341370100339276,
|
|
"grad_norm": 11.44354948885957,
|
|
"learning_rate": 1.0015880653183259e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10170
|
|
},
|
|
{
|
|
"epoch": 0.7348588753338627,
|
|
"grad_norm": 7.4670272071034836,
|
|
"learning_rate": 9.965499757996997e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1356,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10180
|
|
},
|
|
{
|
|
"epoch": 0.7355807406337977,
|
|
"grad_norm": 9.061126614577189,
|
|
"learning_rate": 9.915214332298638e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10190
|
|
},
|
|
{
|
|
"epoch": 0.7363026059337328,
|
|
"grad_norm": 10.142460768085758,
|
|
"learning_rate": 9.865024695402014e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1474,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"epoch": 0.7370244712336678,
|
|
"grad_norm": 9.026651543187556,
|
|
"learning_rate": 9.814931166012694e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1423,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10210
|
|
},
|
|
{
|
|
"epoch": 0.7377463365336028,
|
|
"grad_norm": 11.416603725375351,
|
|
"learning_rate": 9.76493406222594e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1527,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10220
|
|
},
|
|
{
|
|
"epoch": 0.7384682018335379,
|
|
"grad_norm": 9.385956313764396,
|
|
"learning_rate": 9.715033701524756e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1609,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10230
|
|
},
|
|
{
|
|
"epoch": 0.7391900671334729,
|
|
"grad_norm": 12.293936918560657,
|
|
"learning_rate": 9.665230400777793e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10240
|
|
},
|
|
{
|
|
"epoch": 0.7399119324334079,
|
|
"grad_norm": 9.17603345329017,
|
|
"learning_rate": 9.615524476237358e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1684,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10250
|
|
},
|
|
{
|
|
"epoch": 0.7406337977333429,
|
|
"grad_norm": 8.175509864643324,
|
|
"learning_rate": 9.565916243537434e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.159,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10260
|
|
},
|
|
{
|
|
"epoch": 0.741355663033278,
|
|
"grad_norm": 9.728008239435237,
|
|
"learning_rate": 9.51640601769168e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 10270
|
|
},
|
|
{
|
|
"epoch": 0.742077528333213,
|
|
"grad_norm": 9.336449460312915,
|
|
"learning_rate": 9.466994113091359e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1763,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10280
|
|
},
|
|
{
|
|
"epoch": 0.7427993936331481,
|
|
"grad_norm": 6.970907874552232,
|
|
"learning_rate": 9.417680843503426e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1527,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10290
|
|
},
|
|
{
|
|
"epoch": 0.7435212589330831,
|
|
"grad_norm": 8.517413689223407,
|
|
"learning_rate": 9.368466522068494e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1418,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10300
|
|
},
|
|
{
|
|
"epoch": 0.7442431242330181,
|
|
"grad_norm": 8.13932194474774,
|
|
"learning_rate": 9.319351461298847e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1489,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 10310
|
|
},
|
|
{
|
|
"epoch": 0.7449649895329532,
|
|
"grad_norm": 9.845130667757584,
|
|
"learning_rate": 9.270335973076468e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10320
|
|
},
|
|
{
|
|
"epoch": 0.7456868548328882,
|
|
"grad_norm": 8.005007818856038,
|
|
"learning_rate": 9.221420368651045e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10330
|
|
},
|
|
{
|
|
"epoch": 0.7464087201328232,
|
|
"grad_norm": 8.120452643885544,
|
|
"learning_rate": 9.172604958638008e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10340
|
|
},
|
|
{
|
|
"epoch": 0.7471305854327582,
|
|
"grad_norm": 10.68134693575453,
|
|
"learning_rate": 9.123890053016545e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.143,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10350
|
|
},
|
|
{
|
|
"epoch": 0.7478524507326932,
|
|
"grad_norm": 11.395758028840984,
|
|
"learning_rate": 9.07527596112764e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10360
|
|
},
|
|
{
|
|
"epoch": 0.7485743160326284,
|
|
"grad_norm": 8.515447114819407,
|
|
"learning_rate": 9.026762991672105e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10370
|
|
},
|
|
{
|
|
"epoch": 0.7492961813325634,
|
|
"grad_norm": 11.086441240036681,
|
|
"learning_rate": 8.978351452708626e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10380
|
|
},
|
|
{
|
|
"epoch": 0.7500180466324984,
|
|
"grad_norm": 7.288262141466547,
|
|
"learning_rate": 8.930041651651795e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1421,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10390
|
|
},
|
|
{
|
|
"epoch": 0.7507399119324334,
|
|
"grad_norm": 11.30869446863447,
|
|
"learning_rate": 8.881833895270186e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1281,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"epoch": 0.7514617772323684,
|
|
"grad_norm": 10.142597794950875,
|
|
"learning_rate": 8.833728489684341e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.138,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10410
|
|
},
|
|
{
|
|
"epoch": 0.7521836425323035,
|
|
"grad_norm": 8.825883542812928,
|
|
"learning_rate": 8.78572574036493e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10420
|
|
},
|
|
{
|
|
"epoch": 0.7529055078322385,
|
|
"grad_norm": 9.685940029071732,
|
|
"learning_rate": 8.73782595213072e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1498,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10430
|
|
},
|
|
{
|
|
"epoch": 0.7536273731321735,
|
|
"grad_norm": 8.534843301812261,
|
|
"learning_rate": 8.690029429146656e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10440
|
|
},
|
|
{
|
|
"epoch": 0.7543492384321085,
|
|
"grad_norm": 8.26450848095713,
|
|
"learning_rate": 8.642336474922e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1501,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 5.0625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10450
|
|
},
|
|
{
|
|
"epoch": 0.7550711037320436,
|
|
"grad_norm": 7.979518457621037,
|
|
"learning_rate": 8.594747392308288e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10460
|
|
},
|
|
{
|
|
"epoch": 0.7557929690319787,
|
|
"grad_norm": 6.740894768789697,
|
|
"learning_rate": 8.5472624834975e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1188,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10470
|
|
},
|
|
{
|
|
"epoch": 0.7565148343319137,
|
|
"grad_norm": 13.525079704406558,
|
|
"learning_rate": 8.499882050020129e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10480
|
|
},
|
|
{
|
|
"epoch": 0.7572366996318487,
|
|
"grad_norm": 8.368669950131547,
|
|
"learning_rate": 8.4526063927432e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10490
|
|
},
|
|
{
|
|
"epoch": 0.7579585649317837,
|
|
"grad_norm": 6.8452078042054,
|
|
"learning_rate": 8.40543581186843e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1398,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"epoch": 0.7586804302317187,
|
|
"grad_norm": 9.842039787756281,
|
|
"learning_rate": 8.358370606930324e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10510
|
|
},
|
|
{
|
|
"epoch": 0.7594022955316538,
|
|
"grad_norm": 9.548360738442916,
|
|
"learning_rate": 8.311411076794195e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1493,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10520
|
|
},
|
|
{
|
|
"epoch": 0.7601241608315888,
|
|
"grad_norm": 10.51031541236341,
|
|
"learning_rate": 8.264557519654353e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10530
|
|
},
|
|
{
|
|
"epoch": 0.7608460261315239,
|
|
"grad_norm": 8.514510120912588,
|
|
"learning_rate": 8.217810233032168e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1358,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10540
|
|
},
|
|
{
|
|
"epoch": 0.7615678914314589,
|
|
"grad_norm": 11.439380123835884,
|
|
"learning_rate": 8.171169513774189e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10550
|
|
},
|
|
{
|
|
"epoch": 0.7622897567313939,
|
|
"grad_norm": 10.853410558035229,
|
|
"learning_rate": 8.12463565805026e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10560
|
|
},
|
|
{
|
|
"epoch": 0.763011622031329,
|
|
"grad_norm": 10.758545302896312,
|
|
"learning_rate": 8.078208961351637e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10570
|
|
},
|
|
{
|
|
"epoch": 0.763733487331264,
|
|
"grad_norm": 9.106309959022951,
|
|
"learning_rate": 8.031889718489124e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1498,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.75,
|
|
"step": 10580
|
|
},
|
|
{
|
|
"epoch": 0.764455352631199,
|
|
"grad_norm": 9.985703943059816,
|
|
"learning_rate": 7.985678223591155e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10590
|
|
},
|
|
{
|
|
"epoch": 0.765177217931134,
|
|
"grad_norm": 10.95962922741963,
|
|
"learning_rate": 7.939574770102011e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"epoch": 0.765899083231069,
|
|
"grad_norm": 5.8529547106778015,
|
|
"learning_rate": 7.893579650779883e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1209,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10610
|
|
},
|
|
{
|
|
"epoch": 0.7666209485310042,
|
|
"grad_norm": 12.078898463041797,
|
|
"learning_rate": 7.847693157695015e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10620
|
|
},
|
|
{
|
|
"epoch": 0.7673428138309392,
|
|
"grad_norm": 17.665428832633168,
|
|
"learning_rate": 7.801915582227916e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10630
|
|
},
|
|
{
|
|
"epoch": 0.7680646791308742,
|
|
"grad_norm": 10.799427239930818,
|
|
"learning_rate": 7.756247215067444e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10640
|
|
},
|
|
{
|
|
"epoch": 0.7687865444308092,
|
|
"grad_norm": 7.898970963786076,
|
|
"learning_rate": 7.710688346208952e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10650
|
|
},
|
|
{
|
|
"epoch": 0.7695084097307442,
|
|
"grad_norm": 12.877547551422065,
|
|
"learning_rate": 7.665239264952541e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10660
|
|
},
|
|
{
|
|
"epoch": 0.7702302750306793,
|
|
"grad_norm": 9.13069580760062,
|
|
"learning_rate": 7.619900259901097e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1383,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10670
|
|
},
|
|
{
|
|
"epoch": 0.7709521403306143,
|
|
"grad_norm": 7.025486658921553,
|
|
"learning_rate": 7.574671618958544e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1311,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10680
|
|
},
|
|
{
|
|
"epoch": 0.7716740056305493,
|
|
"grad_norm": 8.042577057148172,
|
|
"learning_rate": 7.529553629327994e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1337,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.875,
|
|
"step": 10690
|
|
},
|
|
{
|
|
"epoch": 0.7723958709304843,
|
|
"grad_norm": 9.406345265786733,
|
|
"learning_rate": 7.484546577509918e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1457,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10700
|
|
},
|
|
{
|
|
"epoch": 0.7731177362304194,
|
|
"grad_norm": 11.471896873198808,
|
|
"learning_rate": 7.439650749300322e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1757,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10710
|
|
},
|
|
{
|
|
"epoch": 0.7738396015303545,
|
|
"grad_norm": 8.152571305874515,
|
|
"learning_rate": 7.394866429788945e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.139,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10720
|
|
},
|
|
{
|
|
"epoch": 0.7745614668302895,
|
|
"grad_norm": 8.210195423512335,
|
|
"learning_rate": 7.350193903357444e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10730
|
|
},
|
|
{
|
|
"epoch": 0.7752833321302245,
|
|
"grad_norm": 6.389795872006602,
|
|
"learning_rate": 7.305633453677579e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1468,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.875,
|
|
"step": 10740
|
|
},
|
|
{
|
|
"epoch": 0.7760051974301595,
|
|
"grad_norm": 9.502303401754403,
|
|
"learning_rate": 7.26118536370943e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -1248.0,
|
|
"loss": 0.1385,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -11.0,
|
|
"step": 10750
|
|
},
|
|
{
|
|
"epoch": 0.7767270627300946,
|
|
"grad_norm": 7.527767780807479,
|
|
"learning_rate": 7.21684991569958e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10760
|
|
},
|
|
{
|
|
"epoch": 0.7774489280300296,
|
|
"grad_norm": 13.895751954232171,
|
|
"learning_rate": 7.17262739117934e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10770
|
|
},
|
|
{
|
|
"epoch": 0.7781707933299646,
|
|
"grad_norm": 5.314719639389733,
|
|
"learning_rate": 7.128518070962947e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1362,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 10780
|
|
},
|
|
{
|
|
"epoch": 0.7788926586298996,
|
|
"grad_norm": 7.382521391430954,
|
|
"learning_rate": 7.084522235145796e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1633,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10790
|
|
},
|
|
{
|
|
"epoch": 0.7796145239298347,
|
|
"grad_norm": 8.89077829343609,
|
|
"learning_rate": 7.040640163102646e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.1535,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.9375,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"epoch": 0.7803363892297698,
|
|
"grad_norm": 9.921923991247489,
|
|
"learning_rate": 6.996872133485854e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.161,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10810
|
|
},
|
|
{
|
|
"epoch": 0.7810582545297048,
|
|
"grad_norm": 10.225109832732958,
|
|
"learning_rate": 6.953218424223617e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1486,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10820
|
|
},
|
|
{
|
|
"epoch": 0.7817801198296398,
|
|
"grad_norm": 8.45743164183002,
|
|
"learning_rate": 6.909679312518163e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1594,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 10830
|
|
},
|
|
{
|
|
"epoch": 0.7825019851295748,
|
|
"grad_norm": 8.068263202065939,
|
|
"learning_rate": 6.866255074844046e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1357,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10840
|
|
},
|
|
{
|
|
"epoch": 0.7832238504295098,
|
|
"grad_norm": 7.516647068103794,
|
|
"learning_rate": 6.822945986946385e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10850
|
|
},
|
|
{
|
|
"epoch": 0.7839457157294449,
|
|
"grad_norm": 6.275783146543779,
|
|
"learning_rate": 6.77975232383905e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1288.0,
|
|
"loss": 0.147,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 5.1875,
|
|
"rewards/rejected": -11.1875,
|
|
"step": 10860
|
|
},
|
|
{
|
|
"epoch": 0.78466758102938,
|
|
"grad_norm": 6.33801021737826,
|
|
"learning_rate": 6.736674359802986e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10870
|
|
},
|
|
{
|
|
"epoch": 0.785389446329315,
|
|
"grad_norm": 5.535919923586679,
|
|
"learning_rate": 6.693712368384463e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10880
|
|
},
|
|
{
|
|
"epoch": 0.78611131162925,
|
|
"grad_norm": 8.838224158602769,
|
|
"learning_rate": 6.650866622393281e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1724,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 10890
|
|
},
|
|
{
|
|
"epoch": 0.786833176929185,
|
|
"grad_norm": 11.077227000211735,
|
|
"learning_rate": 6.608137393901106e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1658,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10900
|
|
},
|
|
{
|
|
"epoch": 0.7875550422291201,
|
|
"grad_norm": 8.408163966199039,
|
|
"learning_rate": 6.565524954239709e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 10910
|
|
},
|
|
{
|
|
"epoch": 0.7882769075290551,
|
|
"grad_norm": 6.975124530660155,
|
|
"learning_rate": 6.523029573999247e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1341,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10920
|
|
},
|
|
{
|
|
"epoch": 0.7889987728289901,
|
|
"grad_norm": 9.65061642352629,
|
|
"learning_rate": 6.480651523026548e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1458,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10930
|
|
},
|
|
{
|
|
"epoch": 0.7897206381289251,
|
|
"grad_norm": 7.84205002288234,
|
|
"learning_rate": 6.438391070423396e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1422,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 10940
|
|
},
|
|
{
|
|
"epoch": 0.7904425034288601,
|
|
"grad_norm": 9.668441648479394,
|
|
"learning_rate": 6.396248484544817e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10950
|
|
},
|
|
{
|
|
"epoch": 0.7911643687287953,
|
|
"grad_norm": 14.668012539452555,
|
|
"learning_rate": 6.354224032997391e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -824.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1396,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.5,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10960
|
|
},
|
|
{
|
|
"epoch": 0.7918862340287303,
|
|
"grad_norm": 12.790645204648316,
|
|
"learning_rate": 6.31231798263753e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 10970
|
|
},
|
|
{
|
|
"epoch": 0.7926080993286653,
|
|
"grad_norm": 8.256443608309835,
|
|
"learning_rate": 6.27053059956981e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1357,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 10980
|
|
},
|
|
{
|
|
"epoch": 0.7933299646286003,
|
|
"grad_norm": 7.4278417405838635,
|
|
"learning_rate": 6.228862149145234e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.137,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10990
|
|
},
|
|
{
|
|
"epoch": 0.7940518299285353,
|
|
"grad_norm": 11.30809033177476,
|
|
"learning_rate": 6.187312895959623e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.875,
|
|
"step": 11000
|
|
},
|
|
{
|
|
"epoch": 0.7947736952284704,
|
|
"grad_norm": 8.324969415531836,
|
|
"learning_rate": 6.145883103851876e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1482,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11010
|
|
},
|
|
{
|
|
"epoch": 0.7954955605284054,
|
|
"grad_norm": 7.879621497542356,
|
|
"learning_rate": 6.10457303590228e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1544,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11020
|
|
},
|
|
{
|
|
"epoch": 0.7962174258283404,
|
|
"grad_norm": 13.824352620024788,
|
|
"learning_rate": 6.063382954430921e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.159,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11030
|
|
},
|
|
{
|
|
"epoch": 0.7969392911282754,
|
|
"grad_norm": 9.7321719620388,
|
|
"learning_rate": 6.022313120995942e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11040
|
|
},
|
|
{
|
|
"epoch": 0.7976611564282104,
|
|
"grad_norm": 7.721151928722837,
|
|
"learning_rate": 5.981363796391889e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1234,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11050
|
|
},
|
|
{
|
|
"epoch": 0.7983830217281456,
|
|
"grad_norm": 9.328263463248103,
|
|
"learning_rate": 5.9405352406481226e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11060
|
|
},
|
|
{
|
|
"epoch": 0.7991048870280806,
|
|
"grad_norm": 8.988279736482538,
|
|
"learning_rate": 5.899827713027064e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11070
|
|
},
|
|
{
|
|
"epoch": 0.7998267523280156,
|
|
"grad_norm": 11.860283883793564,
|
|
"learning_rate": 5.859241472022633e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1316,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11080
|
|
},
|
|
{
|
|
"epoch": 0.8005486176279506,
|
|
"grad_norm": 8.339756060333015,
|
|
"learning_rate": 5.818776775358586e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11090
|
|
},
|
|
{
|
|
"epoch": 0.8012704829278856,
|
|
"grad_norm": 11.113384556129915,
|
|
"learning_rate": 5.778433879986835e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1515,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11100
|
|
},
|
|
{
|
|
"epoch": 0.8019923482278207,
|
|
"grad_norm": 10.599463504430258,
|
|
"learning_rate": 5.7382130420858794e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1681,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11110
|
|
},
|
|
{
|
|
"epoch": 0.8027142135277557,
|
|
"grad_norm": 10.696808930709432,
|
|
"learning_rate": 5.698114517059135e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1372,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11120
|
|
},
|
|
{
|
|
"epoch": 0.8034360788276907,
|
|
"grad_norm": 8.944453033412174,
|
|
"learning_rate": 5.658138559533338e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1787,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11130
|
|
},
|
|
{
|
|
"epoch": 0.8041579441276258,
|
|
"grad_norm": 8.977764582899535,
|
|
"learning_rate": 5.6182854233569084e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11140
|
|
},
|
|
{
|
|
"epoch": 0.8048798094275608,
|
|
"grad_norm": 10.126224395800762,
|
|
"learning_rate": 5.578555361598353e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1324,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11150
|
|
},
|
|
{
|
|
"epoch": 0.8056016747274959,
|
|
"grad_norm": 7.398653029233116,
|
|
"learning_rate": 5.538948626544651e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1445,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11160
|
|
},
|
|
{
|
|
"epoch": 0.8063235400274309,
|
|
"grad_norm": 8.672148610321091,
|
|
"learning_rate": 5.4994654696996545e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1573,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11170
|
|
},
|
|
{
|
|
"epoch": 0.8070454053273659,
|
|
"grad_norm": 7.7083477926519,
|
|
"learning_rate": 5.460106141782492e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11180
|
|
},
|
|
{
|
|
"epoch": 0.8077672706273009,
|
|
"grad_norm": 11.545406044029406,
|
|
"learning_rate": 5.420870892725965e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11190
|
|
},
|
|
{
|
|
"epoch": 0.8084891359272359,
|
|
"grad_norm": 10.670951866955587,
|
|
"learning_rate": 5.381759971674987e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11200
|
|
},
|
|
{
|
|
"epoch": 0.809211001227171,
|
|
"grad_norm": 11.225540948573572,
|
|
"learning_rate": 5.3427736269849724e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11210
|
|
},
|
|
{
|
|
"epoch": 0.809932866527106,
|
|
"grad_norm": 7.266681042559748,
|
|
"learning_rate": 5.303912106220285e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11220
|
|
},
|
|
{
|
|
"epoch": 0.8106547318270411,
|
|
"grad_norm": 9.542579667649324,
|
|
"learning_rate": 5.265175656152621e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1671,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11230
|
|
},
|
|
{
|
|
"epoch": 0.8113765971269761,
|
|
"grad_norm": 9.545895584965017,
|
|
"learning_rate": 5.226564522759525e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.40625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11240
|
|
},
|
|
{
|
|
"epoch": 0.8120984624269111,
|
|
"grad_norm": 8.761578973065786,
|
|
"learning_rate": 5.1880789512227434e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1577,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11250
|
|
},
|
|
{
|
|
"epoch": 0.8128203277268462,
|
|
"grad_norm": 10.935949386772302,
|
|
"learning_rate": 5.1497191859267014e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1503,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11260
|
|
},
|
|
{
|
|
"epoch": 0.8135421930267812,
|
|
"grad_norm": 11.542728432624855,
|
|
"learning_rate": 5.111485470456953e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11270
|
|
},
|
|
{
|
|
"epoch": 0.8142640583267162,
|
|
"grad_norm": 9.765483774608478,
|
|
"learning_rate": 5.0733780475986617e-08,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1546,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11280
|
|
},
|
|
{
|
|
"epoch": 0.8149859236266512,
|
|
"grad_norm": 9.322831257835603,
|
|
"learning_rate": 5.035397159334984e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1422,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11290
|
|
},
|
|
{
|
|
"epoch": 0.8157077889265864,
|
|
"grad_norm": 8.623282731830352,
|
|
"learning_rate": 4.997543046845604e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1369,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11300
|
|
},
|
|
{
|
|
"epoch": 0.8164296542265214,
|
|
"grad_norm": 6.745112334659091,
|
|
"learning_rate": 4.959815950505175e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1443,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11310
|
|
},
|
|
{
|
|
"epoch": 0.8171515195264564,
|
|
"grad_norm": 10.676187874451763,
|
|
"learning_rate": 4.922216109881791e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1458,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11320
|
|
},
|
|
{
|
|
"epoch": 0.8178733848263914,
|
|
"grad_norm": 7.184364110256313,
|
|
"learning_rate": 4.8847437637354664e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1548,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11330
|
|
},
|
|
{
|
|
"epoch": 0.8185952501263264,
|
|
"grad_norm": 10.120238987000853,
|
|
"learning_rate": 4.8473991500166234e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1453,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11340
|
|
},
|
|
{
|
|
"epoch": 0.8193171154262615,
|
|
"grad_norm": 11.946412125314207,
|
|
"learning_rate": 4.810182505864585e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1469,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11350
|
|
},
|
|
{
|
|
"epoch": 0.8200389807261965,
|
|
"grad_norm": 12.472625629846199,
|
|
"learning_rate": 4.7730940676060564e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1556,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11360
|
|
},
|
|
{
|
|
"epoch": 0.8207608460261315,
|
|
"grad_norm": 6.685150027611601,
|
|
"learning_rate": 4.736134070753639e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11370
|
|
},
|
|
{
|
|
"epoch": 0.8214827113260665,
|
|
"grad_norm": 10.618261196059562,
|
|
"learning_rate": 4.6993027500043275e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1579,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.9375,
|
|
"step": 11380
|
|
},
|
|
{
|
|
"epoch": 0.8222045766260015,
|
|
"grad_norm": 9.968980323412524,
|
|
"learning_rate": 4.662600339237999e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11390
|
|
},
|
|
{
|
|
"epoch": 0.8229264419259367,
|
|
"grad_norm": 9.132558632733545,
|
|
"learning_rate": 4.6260270715159836e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.161,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11400
|
|
},
|
|
{
|
|
"epoch": 0.8236483072258717,
|
|
"grad_norm": 9.046561088489032,
|
|
"learning_rate": 4.589583179079531e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11410
|
|
},
|
|
{
|
|
"epoch": 0.8243701725258067,
|
|
"grad_norm": 9.600138399413229,
|
|
"learning_rate": 4.553268893348339e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1280.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 5.21875,
|
|
"rewards/rejected": -11.0625,
|
|
"step": 11420
|
|
},
|
|
{
|
|
"epoch": 0.8250920378257417,
|
|
"grad_norm": 11.534256892624052,
|
|
"learning_rate": 4.5170844449191364e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1586,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11430
|
|
},
|
|
{
|
|
"epoch": 0.8258139031256767,
|
|
"grad_norm": 8.265111331997474,
|
|
"learning_rate": 4.481030063564156e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1372,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11440
|
|
},
|
|
{
|
|
"epoch": 0.8265357684256118,
|
|
"grad_norm": 9.696675663745067,
|
|
"learning_rate": 4.445105978229688e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -10.75,
|
|
"step": 11450
|
|
},
|
|
{
|
|
"epoch": 0.8272576337255468,
|
|
"grad_norm": 11.965530383018477,
|
|
"learning_rate": 4.409312417034683e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1372,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11460
|
|
},
|
|
{
|
|
"epoch": 0.8279794990254818,
|
|
"grad_norm": 10.288831340165839,
|
|
"learning_rate": 4.373649607269217e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1517,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11470
|
|
},
|
|
{
|
|
"epoch": 0.8287013643254169,
|
|
"grad_norm": 10.58525488188931,
|
|
"learning_rate": 4.338117775393107e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11480
|
|
},
|
|
{
|
|
"epoch": 0.8294232296253519,
|
|
"grad_norm": 11.407495223286647,
|
|
"learning_rate": 4.3027171470344767e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1333,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11490
|
|
},
|
|
{
|
|
"epoch": 0.830145094925287,
|
|
"grad_norm": 7.549142594426182,
|
|
"learning_rate": 4.267447946988262e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11500
|
|
},
|
|
{
|
|
"epoch": 0.830866960225222,
|
|
"grad_norm": 10.39718086616228,
|
|
"learning_rate": 4.232310399214853e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11510
|
|
},
|
|
{
|
|
"epoch": 0.831588825525157,
|
|
"grad_norm": 7.71946842083615,
|
|
"learning_rate": 4.1973047268386545e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1261,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11520
|
|
},
|
|
{
|
|
"epoch": 0.832310690825092,
|
|
"grad_norm": 11.394005391721311,
|
|
"learning_rate": 4.162431152146631e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11530
|
|
},
|
|
{
|
|
"epoch": 0.833032556125027,
|
|
"grad_norm": 9.621474950872127,
|
|
"learning_rate": 4.127689896586936e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.141,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11540
|
|
},
|
|
{
|
|
"epoch": 0.8337544214249621,
|
|
"grad_norm": 7.232695813686058,
|
|
"learning_rate": 4.093081180767494e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1313,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11550
|
|
},
|
|
{
|
|
"epoch": 0.8344762867248972,
|
|
"grad_norm": 7.421382018126838,
|
|
"learning_rate": 4.0586052244546e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11560
|
|
},
|
|
{
|
|
"epoch": 0.8351981520248322,
|
|
"grad_norm": 9.602867969798325,
|
|
"learning_rate": 4.0242622465715196e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11570
|
|
},
|
|
{
|
|
"epoch": 0.8359200173247672,
|
|
"grad_norm": 9.389336268350176,
|
|
"learning_rate": 3.9900524651970995e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11580
|
|
},
|
|
{
|
|
"epoch": 0.8366418826247022,
|
|
"grad_norm": 9.480201806828463,
|
|
"learning_rate": 3.9559760975643897e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1461,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11590
|
|
},
|
|
{
|
|
"epoch": 0.8373637479246373,
|
|
"grad_norm": 7.3419299237458695,
|
|
"learning_rate": 3.922033360059254e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1649,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11600
|
|
},
|
|
{
|
|
"epoch": 0.8380856132245723,
|
|
"grad_norm": 9.887973431940114,
|
|
"learning_rate": 3.8882244682190094e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11610
|
|
},
|
|
{
|
|
"epoch": 0.8388074785245073,
|
|
"grad_norm": 6.937760256383144,
|
|
"learning_rate": 3.854549636731047e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1391,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.75,
|
|
"step": 11620
|
|
},
|
|
{
|
|
"epoch": 0.8395293438244423,
|
|
"grad_norm": 8.161082030414025,
|
|
"learning_rate": 3.82100907943145e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11630
|
|
},
|
|
{
|
|
"epoch": 0.8402512091243773,
|
|
"grad_norm": 8.796986939445052,
|
|
"learning_rate": 3.7876030093036945e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.75,
|
|
"step": 11640
|
|
},
|
|
{
|
|
"epoch": 0.8409730744243125,
|
|
"grad_norm": 9.161196064107799,
|
|
"learning_rate": 3.7543316384772375e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1537,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11650
|
|
},
|
|
{
|
|
"epoch": 0.8416949397242475,
|
|
"grad_norm": 8.326472553423818,
|
|
"learning_rate": 3.721195178226178e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1482,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11660
|
|
},
|
|
{
|
|
"epoch": 0.8424168050241825,
|
|
"grad_norm": 8.11503963287216,
|
|
"learning_rate": 3.68819383896796e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1451,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11670
|
|
},
|
|
{
|
|
"epoch": 0.8431386703241175,
|
|
"grad_norm": 8.18334342762608,
|
|
"learning_rate": 3.655327830261995e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1527,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11680
|
|
},
|
|
{
|
|
"epoch": 0.8438605356240525,
|
|
"grad_norm": 9.646827096360845,
|
|
"learning_rate": 3.622597360808324e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1656,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11690
|
|
},
|
|
{
|
|
"epoch": 0.8445824009239876,
|
|
"grad_norm": 8.41592945035557,
|
|
"learning_rate": 3.5900026384463324e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.144,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11700
|
|
},
|
|
{
|
|
"epoch": 0.8453042662239226,
|
|
"grad_norm": 11.091054652826791,
|
|
"learning_rate": 3.557543870153393e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11710
|
|
},
|
|
{
|
|
"epoch": 0.8460261315238576,
|
|
"grad_norm": 10.649551216885522,
|
|
"learning_rate": 3.5252212620435764e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1466,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11720
|
|
},
|
|
{
|
|
"epoch": 0.8467479968237926,
|
|
"grad_norm": 10.591948146106109,
|
|
"learning_rate": 3.493035019366328e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1522,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11730
|
|
},
|
|
{
|
|
"epoch": 0.8474698621237277,
|
|
"grad_norm": 10.912246863703778,
|
|
"learning_rate": 3.460985346505169e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11740
|
|
},
|
|
{
|
|
"epoch": 0.8481917274236628,
|
|
"grad_norm": 6.539575387536508,
|
|
"learning_rate": 3.4290724469764e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1443,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11750
|
|
},
|
|
{
|
|
"epoch": 0.8489135927235978,
|
|
"grad_norm": 12.080456229188744,
|
|
"learning_rate": 3.397296523427806e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1408,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11760
|
|
},
|
|
{
|
|
"epoch": 0.8496354580235328,
|
|
"grad_norm": 10.737805994163432,
|
|
"learning_rate": 3.3656577776373706e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1402,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11770
|
|
},
|
|
{
|
|
"epoch": 0.8503573233234678,
|
|
"grad_norm": 8.746433818443641,
|
|
"learning_rate": 3.334156410511993e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11780
|
|
},
|
|
{
|
|
"epoch": 0.8510791886234029,
|
|
"grad_norm": 8.942255435734593,
|
|
"learning_rate": 3.30279262208622e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1355,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11790
|
|
},
|
|
{
|
|
"epoch": 0.8518010539233379,
|
|
"grad_norm": 13.240281042934914,
|
|
"learning_rate": 3.271566611520965e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11800
|
|
},
|
|
{
|
|
"epoch": 0.852522919223273,
|
|
"grad_norm": 7.445169779102206,
|
|
"learning_rate": 3.240478577102254e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1539,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11810
|
|
},
|
|
{
|
|
"epoch": 0.853244784523208,
|
|
"grad_norm": 10.391094523428233,
|
|
"learning_rate": 3.2095287162399396e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11820
|
|
},
|
|
{
|
|
"epoch": 0.853966649823143,
|
|
"grad_norm": 10.97007525410186,
|
|
"learning_rate": 3.178717225466504e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.75,
|
|
"step": 11830
|
|
},
|
|
{
|
|
"epoch": 0.8546885151230781,
|
|
"grad_norm": 10.949419924686783,
|
|
"learning_rate": 3.1480443004357535e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1419,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11840
|
|
},
|
|
{
|
|
"epoch": 0.8554103804230131,
|
|
"grad_norm": 7.949017711437389,
|
|
"learning_rate": 3.117510135921589e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.127,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11850
|
|
},
|
|
{
|
|
"epoch": 0.8561322457229481,
|
|
"grad_norm": 8.211089267034843,
|
|
"learning_rate": 3.087114925816808e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11860
|
|
},
|
|
{
|
|
"epoch": 0.8568541110228831,
|
|
"grad_norm": 7.931702604806061,
|
|
"learning_rate": 3.0568588631318115e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1334,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11870
|
|
},
|
|
{
|
|
"epoch": 0.8575759763228181,
|
|
"grad_norm": 13.782675522555701,
|
|
"learning_rate": 3.026742139993427e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11880
|
|
},
|
|
{
|
|
"epoch": 0.8582978416227532,
|
|
"grad_norm": 8.568533344202447,
|
|
"learning_rate": 2.996764947643685e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.9375,
|
|
"step": 11890
|
|
},
|
|
{
|
|
"epoch": 0.8590197069226883,
|
|
"grad_norm": 12.27253477325102,
|
|
"learning_rate": 2.9669274764385604e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11900
|
|
},
|
|
{
|
|
"epoch": 0.8597415722226233,
|
|
"grad_norm": 8.289657053798905,
|
|
"learning_rate": 2.9372299158468144e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11910
|
|
},
|
|
{
|
|
"epoch": 0.8604634375225583,
|
|
"grad_norm": 10.205485648725467,
|
|
"learning_rate": 2.907672454448784e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1457,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11920
|
|
},
|
|
{
|
|
"epoch": 0.8611853028224933,
|
|
"grad_norm": 9.498983941338254,
|
|
"learning_rate": 2.8782552799351404e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1494,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11930
|
|
},
|
|
{
|
|
"epoch": 0.8619071681224284,
|
|
"grad_norm": 6.074944238826367,
|
|
"learning_rate": 2.8489785791057543e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1479,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11940
|
|
},
|
|
{
|
|
"epoch": 0.8626290334223634,
|
|
"grad_norm": 8.302877488046027,
|
|
"learning_rate": 2.819842537868475e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11950
|
|
},
|
|
{
|
|
"epoch": 0.8633508987222984,
|
|
"grad_norm": 11.61178207342639,
|
|
"learning_rate": 2.7908473412379618e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1614,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 11960
|
|
},
|
|
{
|
|
"epoch": 0.8640727640222334,
|
|
"grad_norm": 9.45795876040694,
|
|
"learning_rate": 2.761993173334509e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1542,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11970
|
|
},
|
|
{
|
|
"epoch": 0.8647946293221684,
|
|
"grad_norm": 11.362932343916869,
|
|
"learning_rate": 2.733280217382869e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1466,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11980
|
|
},
|
|
{
|
|
"epoch": 0.8655164946221036,
|
|
"grad_norm": 10.568145168122776,
|
|
"learning_rate": 2.704708655711102e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1357,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11990
|
|
},
|
|
{
|
|
"epoch": 0.8662383599220386,
|
|
"grad_norm": 6.986633654793124,
|
|
"learning_rate": 2.6762786697494016e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.875,
|
|
"step": 12000
|
|
},
|
|
{
|
|
"epoch": 0.8669602252219736,
|
|
"grad_norm": 7.778882127033462,
|
|
"learning_rate": 2.6479904400289578e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12010
|
|
},
|
|
{
|
|
"epoch": 0.8676820905219086,
|
|
"grad_norm": 8.539754631291549,
|
|
"learning_rate": 2.6198441461808106e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1264.0,
|
|
"loss": 0.1475,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -11.0,
|
|
"step": 12020
|
|
},
|
|
{
|
|
"epoch": 0.8684039558218436,
|
|
"grad_norm": 7.963728288501986,
|
|
"learning_rate": 2.5918399669346808e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1253,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12030
|
|
},
|
|
{
|
|
"epoch": 0.8691258211217787,
|
|
"grad_norm": 10.257823465025604,
|
|
"learning_rate": 2.5639780801178844e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12040
|
|
},
|
|
{
|
|
"epoch": 0.8698476864217137,
|
|
"grad_norm": 8.314704744595538,
|
|
"learning_rate": 2.53625866265417e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -808.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1427,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12050
|
|
},
|
|
{
|
|
"epoch": 0.8705695517216487,
|
|
"grad_norm": 8.499655093782934,
|
|
"learning_rate": 2.508681890562575e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1355,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12060
|
|
},
|
|
{
|
|
"epoch": 0.8712914170215837,
|
|
"grad_norm": 7.835075004860599,
|
|
"learning_rate": 2.4812479389563794e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1513,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12070
|
|
},
|
|
{
|
|
"epoch": 0.8720132823215188,
|
|
"grad_norm": 9.592674313284755,
|
|
"learning_rate": 2.4539569820419213e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1248.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.875,
|
|
"step": 12080
|
|
},
|
|
{
|
|
"epoch": 0.8727351476214539,
|
|
"grad_norm": 7.77863378103726,
|
|
"learning_rate": 2.4268091931175128e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1351,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12090
|
|
},
|
|
{
|
|
"epoch": 0.8734570129213889,
|
|
"grad_norm": 5.640158653910894,
|
|
"learning_rate": 2.3998047445723728e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1468,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12100
|
|
},
|
|
{
|
|
"epoch": 0.8741788782213239,
|
|
"grad_norm": 9.254054430708486,
|
|
"learning_rate": 2.3729438078854748e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1658,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12110
|
|
},
|
|
{
|
|
"epoch": 0.8749007435212589,
|
|
"grad_norm": 12.122038328529602,
|
|
"learning_rate": 2.3462265536245085e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1587,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.4375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 12120
|
|
},
|
|
{
|
|
"epoch": 0.8756226088211939,
|
|
"grad_norm": 5.974867571191328,
|
|
"learning_rate": 2.3196531514447643e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1273,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12130
|
|
},
|
|
{
|
|
"epoch": 0.876344474121129,
|
|
"grad_norm": 9.921544747499848,
|
|
"learning_rate": 2.293223770088079e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1421,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12140
|
|
},
|
|
{
|
|
"epoch": 0.877066339421064,
|
|
"grad_norm": 12.38062445080125,
|
|
"learning_rate": 2.2669385773817467e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1354,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12150
|
|
},
|
|
{
|
|
"epoch": 0.877788204720999,
|
|
"grad_norm": 12.239735356816436,
|
|
"learning_rate": 2.2407977402374545e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.148,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12160
|
|
},
|
|
{
|
|
"epoch": 0.8785100700209341,
|
|
"grad_norm": 9.840653330548708,
|
|
"learning_rate": 2.2148014246502395e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12170
|
|
},
|
|
{
|
|
"epoch": 0.8792319353208691,
|
|
"grad_norm": 9.716661953638562,
|
|
"learning_rate": 2.1889497956974146e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1656,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12180
|
|
},
|
|
{
|
|
"epoch": 0.8799538006208042,
|
|
"grad_norm": 8.417846830214762,
|
|
"learning_rate": 2.1632430175375332e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12190
|
|
},
|
|
{
|
|
"epoch": 0.8806756659207392,
|
|
"grad_norm": 9.941969312589332,
|
|
"learning_rate": 2.1376812534093375e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1339,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12200
|
|
},
|
|
{
|
|
"epoch": 0.8813975312206742,
|
|
"grad_norm": 10.465626887089373,
|
|
"learning_rate": 2.112264665630728e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12210
|
|
},
|
|
{
|
|
"epoch": 0.8821193965206092,
|
|
"grad_norm": 10.02996418403007,
|
|
"learning_rate": 2.0869934155977348e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.1435,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 12220
|
|
},
|
|
{
|
|
"epoch": 0.8828412618205442,
|
|
"grad_norm": 10.447715723395184,
|
|
"learning_rate": 2.0618676637834924e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1388,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12230
|
|
},
|
|
{
|
|
"epoch": 0.8835631271204794,
|
|
"grad_norm": 9.973159448416428,
|
|
"learning_rate": 2.0368875697372028e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12240
|
|
},
|
|
{
|
|
"epoch": 0.8842849924204144,
|
|
"grad_norm": 7.405712078749057,
|
|
"learning_rate": 2.0120532920831436e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12250
|
|
},
|
|
{
|
|
"epoch": 0.8850068577203494,
|
|
"grad_norm": 8.608940672524163,
|
|
"learning_rate": 1.9873649885196742e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.149,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12260
|
|
},
|
|
{
|
|
"epoch": 0.8857287230202844,
|
|
"grad_norm": 8.76111445121565,
|
|
"learning_rate": 1.9628228158181853e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12270
|
|
},
|
|
{
|
|
"epoch": 0.8864505883202195,
|
|
"grad_norm": 9.726249616502288,
|
|
"learning_rate": 1.93842692982214e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1367,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12280
|
|
},
|
|
{
|
|
"epoch": 0.8871724536201545,
|
|
"grad_norm": 9.856991665137961,
|
|
"learning_rate": 1.9141774854461e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12290
|
|
},
|
|
{
|
|
"epoch": 0.8878943189200895,
|
|
"grad_norm": 8.90097517455146,
|
|
"learning_rate": 1.890074636674685e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12300
|
|
},
|
|
{
|
|
"epoch": 0.8886161842200245,
|
|
"grad_norm": 11.01818428730531,
|
|
"learning_rate": 1.8661185365616478e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12310
|
|
},
|
|
{
|
|
"epoch": 0.8893380495199595,
|
|
"grad_norm": 11.263744199520602,
|
|
"learning_rate": 1.842309337228884e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1501,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12320
|
|
},
|
|
{
|
|
"epoch": 0.8900599148198947,
|
|
"grad_norm": 7.792414941928654,
|
|
"learning_rate": 1.818647189865455e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12330
|
|
},
|
|
{
|
|
"epoch": 0.8907817801198297,
|
|
"grad_norm": 8.762492783110734,
|
|
"learning_rate": 1.7951322447266493e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12340
|
|
},
|
|
{
|
|
"epoch": 0.8915036454197647,
|
|
"grad_norm": 12.278119822877175,
|
|
"learning_rate": 1.7717646511330063e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12350
|
|
},
|
|
{
|
|
"epoch": 0.8922255107196997,
|
|
"grad_norm": 8.026983202327546,
|
|
"learning_rate": 1.748544557469392e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12360
|
|
},
|
|
{
|
|
"epoch": 0.8929473760196347,
|
|
"grad_norm": 8.425357221302765,
|
|
"learning_rate": 1.7254721111840287e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12370
|
|
},
|
|
{
|
|
"epoch": 0.8936692413195698,
|
|
"grad_norm": 9.644004550576527,
|
|
"learning_rate": 1.7025474587875872e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.136,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12380
|
|
},
|
|
{
|
|
"epoch": 0.8943911066195048,
|
|
"grad_norm": 7.131646950070454,
|
|
"learning_rate": 1.6797707458522382e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12390
|
|
},
|
|
{
|
|
"epoch": 0.8951129719194398,
|
|
"grad_norm": 8.868708466246948,
|
|
"learning_rate": 1.6571421170107236e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1545,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12400
|
|
},
|
|
{
|
|
"epoch": 0.8958348372193748,
|
|
"grad_norm": 7.487849791891332,
|
|
"learning_rate": 1.6346617159554628e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12410
|
|
},
|
|
{
|
|
"epoch": 0.8965567025193099,
|
|
"grad_norm": 8.438213927485114,
|
|
"learning_rate": 1.6123296854376217e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12420
|
|
},
|
|
{
|
|
"epoch": 0.897278567819245,
|
|
"grad_norm": 10.754318102307618,
|
|
"learning_rate": 1.5901461672661904e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1789,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12430
|
|
},
|
|
{
|
|
"epoch": 0.89800043311918,
|
|
"grad_norm": 8.21147210255012,
|
|
"learning_rate": 1.5681113023071318e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1399,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12440
|
|
},
|
|
{
|
|
"epoch": 0.898722298419115,
|
|
"grad_norm": 8.618776368547858,
|
|
"learning_rate": 1.5462252304824368e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1313,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12450
|
|
},
|
|
{
|
|
"epoch": 0.89944416371905,
|
|
"grad_norm": 8.41385421629852,
|
|
"learning_rate": 1.524488090769252e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12460
|
|
},
|
|
{
|
|
"epoch": 0.900166029018985,
|
|
"grad_norm": 7.873140216517367,
|
|
"learning_rate": 1.5029000211990216e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1449,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12470
|
|
},
|
|
{
|
|
"epoch": 0.9008878943189201,
|
|
"grad_norm": 9.814168221530482,
|
|
"learning_rate": 1.4814611588565701e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1406,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12480
|
|
},
|
|
{
|
|
"epoch": 0.9016097596188551,
|
|
"grad_norm": 10.539841397759815,
|
|
"learning_rate": 1.4601716398792595e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -804.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.171,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12490
|
|
},
|
|
{
|
|
"epoch": 0.9023316249187902,
|
|
"grad_norm": 9.225273086300604,
|
|
"learning_rate": 1.4390315994561253e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.149,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12500
|
|
},
|
|
{
|
|
"epoch": 0.9030534902187252,
|
|
"grad_norm": 8.885167039781129,
|
|
"learning_rate": 1.4180411718269974e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1346,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12510
|
|
},
|
|
{
|
|
"epoch": 0.9037753555186602,
|
|
"grad_norm": 9.145979733096528,
|
|
"learning_rate": 1.3972004902816609e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1515,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12520
|
|
},
|
|
{
|
|
"epoch": 0.9044972208185953,
|
|
"grad_norm": 7.295474662843511,
|
|
"learning_rate": 1.3765096871590248e-08,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1301,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12530
|
|
},
|
|
{
|
|
"epoch": 0.9052190861185303,
|
|
"grad_norm": 11.370111074734503,
|
|
"learning_rate": 1.355968893846246e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1519,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12540
|
|
},
|
|
{
|
|
"epoch": 0.9059409514184653,
|
|
"grad_norm": 11.025095677328915,
|
|
"learning_rate": 1.3355782407779292e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.75,
|
|
"step": 12550
|
|
},
|
|
{
|
|
"epoch": 0.9066628167184003,
|
|
"grad_norm": 6.549589867605967,
|
|
"learning_rate": 1.3153378574352753e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1384,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12560
|
|
},
|
|
{
|
|
"epoch": 0.9073846820183353,
|
|
"grad_norm": 10.439039283166451,
|
|
"learning_rate": 1.2952478723452759e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12570
|
|
},
|
|
{
|
|
"epoch": 0.9081065473182705,
|
|
"grad_norm": 8.066549462633791,
|
|
"learning_rate": 1.2753084130798841e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12580
|
|
},
|
|
{
|
|
"epoch": 0.9088284126182055,
|
|
"grad_norm": 9.16202650125988,
|
|
"learning_rate": 1.2555196062552121e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1475,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12590
|
|
},
|
|
{
|
|
"epoch": 0.9095502779181405,
|
|
"grad_norm": 9.653491629671723,
|
|
"learning_rate": 1.2358815775307257e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.1395,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12600
|
|
},
|
|
{
|
|
"epoch": 0.9102721432180755,
|
|
"grad_norm": 11.463300528930278,
|
|
"learning_rate": 1.2163944516084434e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1248.0,
|
|
"loss": 0.1523,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.9375,
|
|
"step": 12610
|
|
},
|
|
{
|
|
"epoch": 0.9109940085180105,
|
|
"grad_norm": 7.787381760295542,
|
|
"learning_rate": 1.197058352232147e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1429,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12620
|
|
},
|
|
{
|
|
"epoch": 0.9117158738179456,
|
|
"grad_norm": 10.056484595488875,
|
|
"learning_rate": 1.1778734021866022e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12630
|
|
},
|
|
{
|
|
"epoch": 0.9124377391178806,
|
|
"grad_norm": 8.755038077817517,
|
|
"learning_rate": 1.1588397232967561e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1524,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12640
|
|
},
|
|
{
|
|
"epoch": 0.9131596044178156,
|
|
"grad_norm": 7.008042272492635,
|
|
"learning_rate": 1.1399574364269997e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.145,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12650
|
|
},
|
|
{
|
|
"epoch": 0.9138814697177506,
|
|
"grad_norm": 8.324802964335403,
|
|
"learning_rate": 1.1212266614803706e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1378,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12660
|
|
},
|
|
{
|
|
"epoch": 0.9146033350176856,
|
|
"grad_norm": 10.151515852087343,
|
|
"learning_rate": 1.1026475173977978e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1576,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12670
|
|
},
|
|
{
|
|
"epoch": 0.9153252003176208,
|
|
"grad_norm": 10.157754434801875,
|
|
"learning_rate": 1.0842201221573532e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1472,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12680
|
|
},
|
|
{
|
|
"epoch": 0.9160470656175558,
|
|
"grad_norm": 7.591920449981847,
|
|
"learning_rate": 1.0659445927735127e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12690
|
|
},
|
|
{
|
|
"epoch": 0.9167689309174908,
|
|
"grad_norm": 9.794363619142695,
|
|
"learning_rate": 1.0478210452963737e-08,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.143,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12700
|
|
},
|
|
{
|
|
"epoch": 0.9174907962174258,
|
|
"grad_norm": 9.309110081694072,
|
|
"learning_rate": 1.0298495948109665e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12710
|
|
},
|
|
{
|
|
"epoch": 0.9182126615173608,
|
|
"grad_norm": 7.066887867505908,
|
|
"learning_rate": 1.0120303554364912e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1435,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12720
|
|
},
|
|
{
|
|
"epoch": 0.9189345268172959,
|
|
"grad_norm": 11.301604093462545,
|
|
"learning_rate": 9.943634403256046e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1421,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12730
|
|
},
|
|
{
|
|
"epoch": 0.9196563921172309,
|
|
"grad_norm": 9.577802355257152,
|
|
"learning_rate": 9.768489616637038e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1649,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12740
|
|
},
|
|
{
|
|
"epoch": 0.920378257417166,
|
|
"grad_norm": 8.572385841169005,
|
|
"learning_rate": 9.594870306682045e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12750
|
|
},
|
|
{
|
|
"epoch": 0.921100122717101,
|
|
"grad_norm": 10.447641927889913,
|
|
"learning_rate": 9.42277757587842e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1494,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12760
|
|
},
|
|
{
|
|
"epoch": 0.9218219880170361,
|
|
"grad_norm": 9.167052590543541,
|
|
"learning_rate": 9.25221251701977e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1569,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12770
|
|
},
|
|
{
|
|
"epoch": 0.9225438533169711,
|
|
"grad_norm": 9.609226867480382,
|
|
"learning_rate": 9.083176213198874e-09,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1546,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12780
|
|
},
|
|
{
|
|
"epoch": 0.9232657186169061,
|
|
"grad_norm": 12.308594013486106,
|
|
"learning_rate": 8.915669737800835e-09,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12790
|
|
},
|
|
{
|
|
"epoch": 0.9239875839168411,
|
|
"grad_norm": 7.740941902213041,
|
|
"learning_rate": 8.74969415449639e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1305,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12800
|
|
},
|
|
{
|
|
"epoch": 0.9247094492167761,
|
|
"grad_norm": 9.587184945397313,
|
|
"learning_rate": 8.585250517235048e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1525,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12810
|
|
},
|
|
{
|
|
"epoch": 0.9254313145167112,
|
|
"grad_norm": 12.236899096306711,
|
|
"learning_rate": 8.422339870238409e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1401,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12820
|
|
},
|
|
{
|
|
"epoch": 0.9261531798166462,
|
|
"grad_norm": 9.335127467437971,
|
|
"learning_rate": 8.26096324799347e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12830
|
|
},
|
|
{
|
|
"epoch": 0.9268750451165813,
|
|
"grad_norm": 10.11030845129818,
|
|
"learning_rate": 8.101121675246293e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1364,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12840
|
|
},
|
|
{
|
|
"epoch": 0.9275969104165163,
|
|
"grad_norm": 9.949772616020681,
|
|
"learning_rate": 7.942816166995187e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12850
|
|
},
|
|
{
|
|
"epoch": 0.9283187757164513,
|
|
"grad_norm": 11.04520640854637,
|
|
"learning_rate": 7.786047728484429e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1363,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12860
|
|
},
|
|
{
|
|
"epoch": 0.9290406410163864,
|
|
"grad_norm": 10.967381676803216,
|
|
"learning_rate": 7.630817355197994e-09,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12870
|
|
},
|
|
{
|
|
"epoch": 0.9297625063163214,
|
|
"grad_norm": 9.36940835074148,
|
|
"learning_rate": 7.477126032852888e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1423,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12880
|
|
},
|
|
{
|
|
"epoch": 0.9304843716162564,
|
|
"grad_norm": 6.9344949791967885,
|
|
"learning_rate": 7.324974737393241e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1347,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12890
|
|
},
|
|
{
|
|
"epoch": 0.9312062369161914,
|
|
"grad_norm": 8.914498439457502,
|
|
"learning_rate": 7.174364434984009e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1546,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12900
|
|
},
|
|
{
|
|
"epoch": 0.9319281022161264,
|
|
"grad_norm": 11.255791378742503,
|
|
"learning_rate": 7.025296082004667e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12910
|
|
},
|
|
{
|
|
"epoch": 0.9326499675160616,
|
|
"grad_norm": 10.43918973974646,
|
|
"learning_rate": 6.8777706250433274e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12920
|
|
},
|
|
{
|
|
"epoch": 0.9333718328159966,
|
|
"grad_norm": 9.338539505172553,
|
|
"learning_rate": 6.731789000890775e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12930
|
|
},
|
|
{
|
|
"epoch": 0.9340936981159316,
|
|
"grad_norm": 8.36112263605316,
|
|
"learning_rate": 6.587352136534219e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1377,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 12940
|
|
},
|
|
{
|
|
"epoch": 0.9348155634158666,
|
|
"grad_norm": 8.797834148383416,
|
|
"learning_rate": 6.444460949151714e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12950
|
|
},
|
|
{
|
|
"epoch": 0.9355374287158016,
|
|
"grad_norm": 11.053337065925593,
|
|
"learning_rate": 6.303116346106224e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1535,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12960
|
|
},
|
|
{
|
|
"epoch": 0.9362592940157367,
|
|
"grad_norm": 11.276613104833107,
|
|
"learning_rate": 6.163319224939872e-09,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1325,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12970
|
|
},
|
|
{
|
|
"epoch": 0.9369811593156717,
|
|
"grad_norm": 8.755018080988055,
|
|
"learning_rate": 6.025070473368144e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1502,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12980
|
|
},
|
|
{
|
|
"epoch": 0.9377030246156067,
|
|
"grad_norm": 9.43774867419776,
|
|
"learning_rate": 5.888370969274442e-09,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1572,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12990
|
|
},
|
|
{
|
|
"epoch": 0.9384248899155417,
|
|
"grad_norm": 9.85434140441404,
|
|
"learning_rate": 5.7532215807043486e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1319,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13000
|
|
},
|
|
{
|
|
"epoch": 0.9391467552154767,
|
|
"grad_norm": 7.193797068429037,
|
|
"learning_rate": 5.6196231658601764e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -812.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1416,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.46875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 13010
|
|
},
|
|
{
|
|
"epoch": 0.9398686205154119,
|
|
"grad_norm": 12.631151758721385,
|
|
"learning_rate": 5.48757657309551e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1462,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13020
|
|
},
|
|
{
|
|
"epoch": 0.9405904858153469,
|
|
"grad_norm": 5.307134460751131,
|
|
"learning_rate": 5.357082640909866e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1358,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13030
|
|
},
|
|
{
|
|
"epoch": 0.9413123511152819,
|
|
"grad_norm": 10.087417447571848,
|
|
"learning_rate": 5.22814219794318e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 13040
|
|
},
|
|
{
|
|
"epoch": 0.9420342164152169,
|
|
"grad_norm": 7.00443265106439,
|
|
"learning_rate": 5.10075606297089e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 13050
|
|
},
|
|
{
|
|
"epoch": 0.9427560817151519,
|
|
"grad_norm": 7.793436936582348,
|
|
"learning_rate": 4.974925044898437e-09,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1479,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13060
|
|
},
|
|
{
|
|
"epoch": 0.943477947015087,
|
|
"grad_norm": 6.458652354786663,
|
|
"learning_rate": 4.850649942756135e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1339,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13070
|
|
},
|
|
{
|
|
"epoch": 0.944199812315022,
|
|
"grad_norm": 7.868041435801801,
|
|
"learning_rate": 4.727931545694397e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1424,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13080
|
|
},
|
|
{
|
|
"epoch": 0.944921677614957,
|
|
"grad_norm": 8.264459996035683,
|
|
"learning_rate": 4.606770632978374e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13090
|
|
},
|
|
{
|
|
"epoch": 0.945643542914892,
|
|
"grad_norm": 8.88295658076357,
|
|
"learning_rate": 4.4871679739831304e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13100
|
|
},
|
|
{
|
|
"epoch": 0.9463654082148271,
|
|
"grad_norm": 20.8259081865564,
|
|
"learning_rate": 4.369124328188839e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13110
|
|
},
|
|
{
|
|
"epoch": 0.9470872735147622,
|
|
"grad_norm": 7.245245477976355,
|
|
"learning_rate": 4.252640445175898e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1335,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13120
|
|
},
|
|
{
|
|
"epoch": 0.9478091388146972,
|
|
"grad_norm": 9.917996431238413,
|
|
"learning_rate": 4.1377170646201555e-09,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 13130
|
|
},
|
|
{
|
|
"epoch": 0.9485310041146322,
|
|
"grad_norm": 11.746041159172476,
|
|
"learning_rate": 4.024354916288192e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.17,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13140
|
|
},
|
|
{
|
|
"epoch": 0.9492528694145672,
|
|
"grad_norm": 7.791108577213938,
|
|
"learning_rate": 3.912554720032796e-09,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13150
|
|
},
|
|
{
|
|
"epoch": 0.9499747347145022,
|
|
"grad_norm": 7.770825664424211,
|
|
"learning_rate": 3.8023171857882456e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1343,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13160
|
|
},
|
|
{
|
|
"epoch": 0.9506966000144373,
|
|
"grad_norm": 7.654311704309095,
|
|
"learning_rate": 3.693643013565867e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13170
|
|
},
|
|
{
|
|
"epoch": 0.9514184653143724,
|
|
"grad_norm": 10.335438911558372,
|
|
"learning_rate": 3.586532893449706e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13180
|
|
},
|
|
{
|
|
"epoch": 0.9521403306143074,
|
|
"grad_norm": 9.99678225643105,
|
|
"learning_rate": 3.4809875055918923e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1307,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13190
|
|
},
|
|
{
|
|
"epoch": 0.9528621959142424,
|
|
"grad_norm": 6.490173084222523,
|
|
"learning_rate": 3.3770075202085304e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13200
|
|
},
|
|
{
|
|
"epoch": 0.9535840612141774,
|
|
"grad_norm": 8.060395264263372,
|
|
"learning_rate": 3.274593597575398e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13210
|
|
},
|
|
{
|
|
"epoch": 0.9543059265141125,
|
|
"grad_norm": 7.41254192877576,
|
|
"learning_rate": 3.173746388023729e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1248.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 13220
|
|
},
|
|
{
|
|
"epoch": 0.9550277918140475,
|
|
"grad_norm": 11.024215888855808,
|
|
"learning_rate": 3.074466531935993e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13230
|
|
},
|
|
{
|
|
"epoch": 0.9557496571139825,
|
|
"grad_norm": 11.151738805917098,
|
|
"learning_rate": 2.976754659742037e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1603,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13240
|
|
},
|
|
{
|
|
"epoch": 0.9564715224139175,
|
|
"grad_norm": 6.773241527202388,
|
|
"learning_rate": 2.88061139191495e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1285,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13250
|
|
},
|
|
{
|
|
"epoch": 0.9571933877138527,
|
|
"grad_norm": 15.814195118042852,
|
|
"learning_rate": 2.7860373389670398e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13260
|
|
},
|
|
{
|
|
"epoch": 0.9579152530137877,
|
|
"grad_norm": 10.16676403861108,
|
|
"learning_rate": 2.693033101446196e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1496,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13270
|
|
},
|
|
{
|
|
"epoch": 0.9586371183137227,
|
|
"grad_norm": 8.485290989034366,
|
|
"learning_rate": 2.6015992699318366e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1261,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13280
|
|
},
|
|
{
|
|
"epoch": 0.9593589836136577,
|
|
"grad_norm": 9.904715194680257,
|
|
"learning_rate": 2.5117364250312744e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13290
|
|
},
|
|
{
|
|
"epoch": 0.9600808489135927,
|
|
"grad_norm": 12.293208072952874,
|
|
"learning_rate": 2.4234451373761068e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1248.0,
|
|
"loss": 0.1576,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.9375,
|
|
"step": 13300
|
|
},
|
|
{
|
|
"epoch": 0.9608027142135278,
|
|
"grad_norm": 9.583806839385215,
|
|
"learning_rate": 2.336725967618358e-09,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.154,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13310
|
|
},
|
|
{
|
|
"epoch": 0.9615245795134628,
|
|
"grad_norm": 9.704485535689619,
|
|
"learning_rate": 2.2515794664271502e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1403,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13320
|
|
},
|
|
{
|
|
"epoch": 0.9622464448133978,
|
|
"grad_norm": 10.451439623478995,
|
|
"learning_rate": 2.168006174485121e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1545,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13330
|
|
},
|
|
{
|
|
"epoch": 0.9629683101133328,
|
|
"grad_norm": 8.838700241798461,
|
|
"learning_rate": 2.0860066224848983e-09,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1344,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13340
|
|
},
|
|
{
|
|
"epoch": 0.9636901754132678,
|
|
"grad_norm": 10.958902722065448,
|
|
"learning_rate": 2.0055813311259383e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13350
|
|
},
|
|
{
|
|
"epoch": 0.964412040713203,
|
|
"grad_norm": 6.09181790905039,
|
|
"learning_rate": 1.926730811111027e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1414,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13360
|
|
},
|
|
{
|
|
"epoch": 0.965133906013138,
|
|
"grad_norm": 9.88628572992933,
|
|
"learning_rate": 1.849455563143143e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13370
|
|
},
|
|
{
|
|
"epoch": 0.965855771313073,
|
|
"grad_norm": 13.53292753569823,
|
|
"learning_rate": 1.773756077922156e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13380
|
|
},
|
|
{
|
|
"epoch": 0.966577636613008,
|
|
"grad_norm": 8.793000871412223,
|
|
"learning_rate": 1.69963283614194e-09,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.155,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13390
|
|
},
|
|
{
|
|
"epoch": 0.967299501912943,
|
|
"grad_norm": 8.92187666322989,
|
|
"learning_rate": 1.6270863084870967e-09,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1348,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13400
|
|
},
|
|
{
|
|
"epoch": 0.9680213672128781,
|
|
"grad_norm": 7.727103982802486,
|
|
"learning_rate": 1.5561169556300157e-09,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13410
|
|
},
|
|
{
|
|
"epoch": 0.9687432325128131,
|
|
"grad_norm": 10.615949449025017,
|
|
"learning_rate": 1.4867252282280974e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1412,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13420
|
|
},
|
|
{
|
|
"epoch": 0.9694650978127481,
|
|
"grad_norm": 8.424515914002487,
|
|
"learning_rate": 1.4189115669206719e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1342,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13430
|
|
},
|
|
{
|
|
"epoch": 0.9701869631126832,
|
|
"grad_norm": 9.868921088219077,
|
|
"learning_rate": 1.3526764023263082e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13440
|
|
},
|
|
{
|
|
"epoch": 0.9709088284126182,
|
|
"grad_norm": 12.402501755945687,
|
|
"learning_rate": 1.288020155040176e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1346,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13450
|
|
},
|
|
{
|
|
"epoch": 0.9716306937125533,
|
|
"grad_norm": 5.426424943097187,
|
|
"learning_rate": 1.2249432356311874e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13460
|
|
},
|
|
{
|
|
"epoch": 0.9723525590124883,
|
|
"grad_norm": 9.000433341065145,
|
|
"learning_rate": 1.163446044639499e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1623,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 13470
|
|
},
|
|
{
|
|
"epoch": 0.9730744243124233,
|
|
"grad_norm": 8.594542564403534,
|
|
"learning_rate": 1.103528972573986e-09,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13480
|
|
},
|
|
{
|
|
"epoch": 0.9737962896123583,
|
|
"grad_norm": 11.926460876248191,
|
|
"learning_rate": 1.045192399909689e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 13490
|
|
},
|
|
{
|
|
"epoch": 0.9745181549122933,
|
|
"grad_norm": 8.469305025697546,
|
|
"learning_rate": 9.884366970853986e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1545,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13500
|
|
},
|
|
{
|
|
"epoch": 0.9752400202122284,
|
|
"grad_norm": 9.638473562645952,
|
|
"learning_rate": 9.332622245014355e-10,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13510
|
|
},
|
|
{
|
|
"epoch": 0.9759618855121635,
|
|
"grad_norm": 10.982840589873115,
|
|
"learning_rate": 8.796693325171522e-10,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1414,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13520
|
|
},
|
|
{
|
|
"epoch": 0.9766837508120985,
|
|
"grad_norm": 9.160859372694532,
|
|
"learning_rate": 8.276583614489352e-10,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1397,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13530
|
|
},
|
|
{
|
|
"epoch": 0.9774056161120335,
|
|
"grad_norm": 11.936529679612834,
|
|
"learning_rate": 7.772296415678447e-10,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -796.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1344,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.3125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 13540
|
|
},
|
|
{
|
|
"epoch": 0.9781274814119685,
|
|
"grad_norm": 7.009572464668358,
|
|
"learning_rate": 7.283834930976451e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1445,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13550
|
|
},
|
|
{
|
|
"epoch": 0.9788493467119036,
|
|
"grad_norm": 7.948146385621628,
|
|
"learning_rate": 6.811202262126947e-10,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.1409,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13560
|
|
},
|
|
{
|
|
"epoch": 0.9795712120118386,
|
|
"grad_norm": 8.433006482064645,
|
|
"learning_rate": 6.354401410360588e-10,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1257,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13570
|
|
},
|
|
{
|
|
"epoch": 0.9802930773117736,
|
|
"grad_norm": 12.067512536747236,
|
|
"learning_rate": 5.913435276374834e-10,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13580
|
|
},
|
|
{
|
|
"epoch": 0.9810149426117086,
|
|
"grad_norm": 8.546742612397637,
|
|
"learning_rate": 5.488306660317299e-10,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 13590
|
|
},
|
|
{
|
|
"epoch": 0.9817368079116436,
|
|
"grad_norm": 10.59407530013682,
|
|
"learning_rate": 5.079018261766044e-10,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13600
|
|
},
|
|
{
|
|
"epoch": 0.9824586732115788,
|
|
"grad_norm": 11.513879858085009,
|
|
"learning_rate": 4.685572679713478e-10,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13610
|
|
},
|
|
{
|
|
"epoch": 0.9831805385115138,
|
|
"grad_norm": 11.717526217203766,
|
|
"learning_rate": 4.3079724125499875e-10,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -800.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13620
|
|
},
|
|
{
|
|
"epoch": 0.9839024038114488,
|
|
"grad_norm": 8.065660623179564,
|
|
"learning_rate": 3.9462198580475505e-10,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 13630
|
|
},
|
|
{
|
|
"epoch": 0.9846242691113838,
|
|
"grad_norm": 11.648253958259497,
|
|
"learning_rate": 3.6003173133447585e-10,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13640
|
|
},
|
|
{
|
|
"epoch": 0.9853461344113188,
|
|
"grad_norm": 12.43000507016397,
|
|
"learning_rate": 3.270266974932101e-10,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1476,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13650
|
|
},
|
|
{
|
|
"epoch": 0.9860679997112539,
|
|
"grad_norm": 10.176611290026623,
|
|
"learning_rate": 2.956070938638644e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.875,
|
|
"step": 13660
|
|
},
|
|
{
|
|
"epoch": 0.9867898650111889,
|
|
"grad_norm": 8.384903385562643,
|
|
"learning_rate": 2.6577311996175964e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13670
|
|
},
|
|
{
|
|
"epoch": 0.9875117303111239,
|
|
"grad_norm": 7.9305338057329005,
|
|
"learning_rate": 2.3752496523343767e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1398,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13680
|
|
},
|
|
{
|
|
"epoch": 0.988233595611059,
|
|
"grad_norm": 12.291251192914357,
|
|
"learning_rate": 2.1086280905543984e-10,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1289,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13690
|
|
},
|
|
{
|
|
"epoch": 0.988955460910994,
|
|
"grad_norm": 6.115365197215902,
|
|
"learning_rate": 1.8578682073322472e-10,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.128,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13700
|
|
},
|
|
{
|
|
"epoch": 0.9896773262109291,
|
|
"grad_norm": 11.676458111445147,
|
|
"learning_rate": 1.6229715950000221e-10,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1443,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.75,
|
|
"step": 13710
|
|
},
|
|
{
|
|
"epoch": 0.9903991915108641,
|
|
"grad_norm": 8.892462780201626,
|
|
"learning_rate": 1.4039397451573455e-10,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1338,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13720
|
|
},
|
|
{
|
|
"epoch": 0.9911210568107991,
|
|
"grad_norm": 8.896892206063134,
|
|
"learning_rate": 1.2007740486624785e-10,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1626,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13730
|
|
},
|
|
{
|
|
"epoch": 0.9918429221107341,
|
|
"grad_norm": 8.41403414937158,
|
|
"learning_rate": 1.0134757956234418e-10,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1526,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13740
|
|
},
|
|
{
|
|
"epoch": 0.9925647874106691,
|
|
"grad_norm": 8.437786909628086,
|
|
"learning_rate": 8.420461753891327e-11,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13750
|
|
},
|
|
{
|
|
"epoch": 0.9932866527106042,
|
|
"grad_norm": 9.160480007480118,
|
|
"learning_rate": 6.864862765421087e-11,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13760
|
|
},
|
|
{
|
|
"epoch": 0.9940085180105392,
|
|
"grad_norm": 11.808273487978743,
|
|
"learning_rate": 5.4679708689248095e-11,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1254,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13770
|
|
},
|
|
{
|
|
"epoch": 0.9947303833104743,
|
|
"grad_norm": 11.167829268938808,
|
|
"learning_rate": 4.2297949346986606e-11,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1435,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13780
|
|
},
|
|
{
|
|
"epoch": 0.9954522486104093,
|
|
"grad_norm": 8.328945950486863,
|
|
"learning_rate": 3.150342825197771e-11,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1432,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13790
|
|
},
|
|
{
|
|
"epoch": 0.9961741139103444,
|
|
"grad_norm": 8.527974081958734,
|
|
"learning_rate": 2.2296213949696273e-11,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13800
|
|
},
|
|
{
|
|
"epoch": 0.9968959792102794,
|
|
"grad_norm": 5.882710049110608,
|
|
"learning_rate": 1.4676364906235362e-11,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13810
|
|
},
|
|
{
|
|
"epoch": 0.9976178445102144,
|
|
"grad_norm": 8.19009325593564,
|
|
"learning_rate": 8.643929507834435e-12,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.625,
|
|
"step": 13820
|
|
},
|
|
{
|
|
"epoch": 0.9983397098101494,
|
|
"grad_norm": 8.07778039690577,
|
|
"learning_rate": 4.198946060601782e-12,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1558,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13830
|
|
},
|
|
{
|
|
"epoch": 0.9990615751100844,
|
|
"grad_norm": 9.049466464418751,
|
|
"learning_rate": 1.3414427903202242e-12,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.139,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 13840
|
|
},
|
|
{
|
|
"epoch": 0.9997834404100195,
|
|
"grad_norm": 7.329571223136626,
|
|
"learning_rate": 7.143784222507499e-14,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1357,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13850
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 13853,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.0,
|
|
"train_runtime": 39.4392,
|
|
"train_samples_per_second": 44959.109,
|
|
"train_steps_per_second": 351.249
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 13853,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 5000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|