15081 lines
444 KiB
JSON
15081 lines
444 KiB
JSON
{
|
|
"best_metric": 0.21195410192012787,
|
|
"best_model_checkpoint": "models/qwen2.5-3b-dpo-finegrained/checkpoint-10000",
|
|
"epoch": 0.7218652999350321,
|
|
"eval_steps": 5000,
|
|
"global_step": 10000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 7.218652999350322e-05,
|
|
"grad_norm": 1.852354340577762,
|
|
"learning_rate": 3.6075036075036073e-10,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -1.6015625,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0007218652999350321,
|
|
"grad_norm": 1.998199384338708,
|
|
"learning_rate": 3.6075036075036073e-09,
|
|
"logits/chosen": -2.265625,
|
|
"logits/rejected": -1.953125,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -143.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.2638888955116272,
|
|
"rewards/chosen": 0.000766754150390625,
|
|
"rewards/margins": 0.000606536865234375,
|
|
"rewards/rejected": 0.000156402587890625,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0014437305998700643,
|
|
"grad_norm": 1.859902408963682,
|
|
"learning_rate": 7.215007215007215e-09,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.001129150390625,
|
|
"rewards/margins": 0.00061798095703125,
|
|
"rewards/rejected": 0.000507354736328125,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0021655958998050965,
|
|
"grad_norm": 1.800945636929482,
|
|
"learning_rate": 1.0822510822510822e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -148.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.3375000059604645,
|
|
"rewards/chosen": 0.000640869140625,
|
|
"rewards/margins": -0.0003757476806640625,
|
|
"rewards/rejected": 0.00101470947265625,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.0028874611997401285,
|
|
"grad_norm": 1.9512697515276394,
|
|
"learning_rate": 1.443001443001443e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.2874999940395355,
|
|
"rewards/chosen": 0.00010967254638671875,
|
|
"rewards/margins": 0.0003910064697265625,
|
|
"rewards/rejected": -0.00028228759765625,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.0036093264996751606,
|
|
"grad_norm": 1.9685077450218404,
|
|
"learning_rate": 1.8037518037518035e-08,
|
|
"logits/chosen": -2.21875,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -143.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.3187499940395355,
|
|
"rewards/chosen": 9.441375732421875e-05,
|
|
"rewards/margins": 0.0001239776611328125,
|
|
"rewards/rejected": -3.0159950256347656e-05,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.004331191799610193,
|
|
"grad_norm": 1.9944209672205195,
|
|
"learning_rate": 2.1645021645021644e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -160.0,
|
|
"logps/rejected": -136.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": -0.000232696533203125,
|
|
"rewards/margins": 0.00102996826171875,
|
|
"rewards/rejected": -0.0012664794921875,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.005053057099545225,
|
|
"grad_norm": 1.8602055997261977,
|
|
"learning_rate": 2.525252525252525e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.8984375,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.3375000059604645,
|
|
"rewards/chosen": 0.000438690185546875,
|
|
"rewards/margins": 0.00054931640625,
|
|
"rewards/rejected": -0.00010919570922851562,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.005774922399480257,
|
|
"grad_norm": 1.7129972696287459,
|
|
"learning_rate": 2.886002886002886e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -170.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.000579833984375,
|
|
"rewards/margins": 0.0004520416259765625,
|
|
"rewards/rejected": 0.0001239776611328125,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.006496787699415289,
|
|
"grad_norm": 1.8246478253492482,
|
|
"learning_rate": 3.246753246753246e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -178.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 0.000469207763671875,
|
|
"rewards/margins": 0.0001888275146484375,
|
|
"rewards/rejected": 0.0002803802490234375,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.007218652999350321,
|
|
"grad_norm": 1.7657315202986226,
|
|
"learning_rate": 3.607503607503607e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3062500059604645,
|
|
"rewards/chosen": 0.000797271728515625,
|
|
"rewards/margins": -0.0003910064697265625,
|
|
"rewards/rejected": 0.001190185546875,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.007940518299285354,
|
|
"grad_norm": 2.0123631368082844,
|
|
"learning_rate": 3.968253968253968e-08,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.9296875,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -140.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.35624998807907104,
|
|
"rewards/chosen": 0.000186920166015625,
|
|
"rewards/margins": 0.00018787384033203125,
|
|
"rewards/rejected": 1.1473894119262695e-06,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.008662383599220386,
|
|
"grad_norm": 1.9667654175700489,
|
|
"learning_rate": 4.329004329004329e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.33125001192092896,
|
|
"rewards/chosen": 0.00154876708984375,
|
|
"rewards/margins": -0.00103759765625,
|
|
"rewards/rejected": 0.0025787353515625,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.009384248899155418,
|
|
"grad_norm": 1.7516436962452608,
|
|
"learning_rate": 4.68975468975469e-08,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.4312500059604645,
|
|
"rewards/chosen": 0.0027313232421875,
|
|
"rewards/margins": 0.00164031982421875,
|
|
"rewards/rejected": 0.0010986328125,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.01010611419909045,
|
|
"grad_norm": 2.0629971445302435,
|
|
"learning_rate": 5.05050505050505e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -167.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6921,
|
|
"rewards/accuracies": 0.39375001192092896,
|
|
"rewards/chosen": 0.0018768310546875,
|
|
"rewards/margins": 0.000438690185546875,
|
|
"rewards/rejected": 0.00144195556640625,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.010827979499025482,
|
|
"grad_norm": 2.0455409873804205,
|
|
"learning_rate": 5.411255411255411e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.40625,
|
|
"rewards/chosen": 0.001739501953125,
|
|
"rewards/margins": 5.435943603515625e-05,
|
|
"rewards/rejected": 0.001678466796875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.011549844798960514,
|
|
"grad_norm": 2.003008491192784,
|
|
"learning_rate": 5.772005772005772e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -1.8359375,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.6919,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.0026702880859375,
|
|
"rewards/margins": 0.00093841552734375,
|
|
"rewards/rejected": 0.00173187255859375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.012271710098895546,
|
|
"grad_norm": 1.8970983946398985,
|
|
"learning_rate": 6.132756132756133e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -1.9921875,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6915,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.00445556640625,
|
|
"rewards/margins": 0.00396728515625,
|
|
"rewards/rejected": 0.0004787445068359375,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.012993575398830578,
|
|
"grad_norm": 1.9000812663628288,
|
|
"learning_rate": 6.493506493506492e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -184.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6913,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.004791259765625,
|
|
"rewards/margins": 0.00274658203125,
|
|
"rewards/rejected": 0.0020294189453125,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.01371544069876561,
|
|
"grad_norm": 1.7694192875136483,
|
|
"learning_rate": 6.854256854256854e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.007598876953125,
|
|
"rewards/margins": 0.003387451171875,
|
|
"rewards/rejected": 0.00421142578125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.014437305998700642,
|
|
"grad_norm": 1.8022914272788901,
|
|
"learning_rate": 7.215007215007214e-08,
|
|
"logits/chosen": -2.296875,
|
|
"logits/rejected": -1.890625,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6907,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": 0.007354736328125,
|
|
"rewards/margins": 0.005584716796875,
|
|
"rewards/rejected": 0.0017547607421875,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.015159171298635674,
|
|
"grad_norm": 1.7754509638333658,
|
|
"learning_rate": 7.575757575757576e-08,
|
|
"logits/chosen": -2.328125,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6908,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.007537841796875,
|
|
"rewards/margins": 0.0034942626953125,
|
|
"rewards/rejected": 0.0040283203125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.015881036598570708,
|
|
"grad_norm": 1.8713087698233093,
|
|
"learning_rate": 7.936507936507936e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -1.8984375,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.00860595703125,
|
|
"rewards/margins": 0.00469970703125,
|
|
"rewards/rejected": 0.00390625,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.01660290189850574,
|
|
"grad_norm": 1.7352700913418746,
|
|
"learning_rate": 8.297258297258297e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -150.0,
|
|
"loss": 0.6897,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 0.01153564453125,
|
|
"rewards/margins": 0.00640869140625,
|
|
"rewards/rejected": 0.005157470703125,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.017324767198440772,
|
|
"grad_norm": 1.923679325098434,
|
|
"learning_rate": 8.658008658008658e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6889,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.01507568359375,
|
|
"rewards/margins": 0.00933837890625,
|
|
"rewards/rejected": 0.005706787109375,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.018046632498375802,
|
|
"grad_norm": 1.9336109183415302,
|
|
"learning_rate": 9.018759018759018e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -198.0,
|
|
"logps/rejected": -190.0,
|
|
"loss": 0.6883,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.015625,
|
|
"rewards/margins": 0.01055908203125,
|
|
"rewards/rejected": 0.005035400390625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.018768497798310836,
|
|
"grad_norm": 1.7118991587292718,
|
|
"learning_rate": 9.37950937950938e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -1.9921875,
|
|
"logps/chosen": -148.0,
|
|
"logps/rejected": -164.0,
|
|
"loss": 0.6873,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": 0.0172119140625,
|
|
"rewards/margins": 0.012451171875,
|
|
"rewards/rejected": 0.00469970703125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.019490363098245866,
|
|
"grad_norm": 2.0498889647791634,
|
|
"learning_rate": 9.74025974025974e-08,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -150.0,
|
|
"loss": 0.6875,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.00982666015625,
|
|
"rewards/rejected": 0.006195068359375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.0202122283981809,
|
|
"grad_norm": 1.7948780770940682,
|
|
"learning_rate": 1.01010101010101e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -170.0,
|
|
"loss": 0.6865,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": 0.018310546875,
|
|
"rewards/margins": 0.01513671875,
|
|
"rewards/rejected": 0.0032196044921875,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.02093409369811593,
|
|
"grad_norm": 1.7485728753360685,
|
|
"learning_rate": 1.0461760461760462e-07,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -173.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6865,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": 0.0172119140625,
|
|
"rewards/margins": 0.0089111328125,
|
|
"rewards/rejected": 0.0084228515625,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.021655958998050964,
|
|
"grad_norm": 1.7752446010354384,
|
|
"learning_rate": 1.0822510822510822e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6857,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0238037109375,
|
|
"rewards/margins": 0.015625,
|
|
"rewards/rejected": 0.00823974609375,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.022377824297985995,
|
|
"grad_norm": 1.9527079542055057,
|
|
"learning_rate": 1.1183261183261184e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -148.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6845,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.0203857421875,
|
|
"rewards/margins": 0.0184326171875,
|
|
"rewards/rejected": 0.00201416015625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.02309968959792103,
|
|
"grad_norm": 1.798683396295616,
|
|
"learning_rate": 1.1544011544011543e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6844,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 0.02197265625,
|
|
"rewards/margins": 0.0184326171875,
|
|
"rewards/rejected": 0.003570556640625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.02382155489785606,
|
|
"grad_norm": 1.822086025310402,
|
|
"learning_rate": 1.1904761904761903e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.9765625,
|
|
"logps/chosen": -159.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6816,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 0.02490234375,
|
|
"rewards/margins": 0.0225830078125,
|
|
"rewards/rejected": 0.002349853515625,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.024543420197791092,
|
|
"grad_norm": 1.8205804931965275,
|
|
"learning_rate": 1.2265512265512265e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6809,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0240478515625,
|
|
"rewards/margins": 0.02392578125,
|
|
"rewards/rejected": 4.839897155761719e-05,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.025265285497726123,
|
|
"grad_norm": 2.015225469187424,
|
|
"learning_rate": 1.2626262626262626e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -138.0,
|
|
"loss": 0.6787,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": 0.023193359375,
|
|
"rewards/margins": 0.02880859375,
|
|
"rewards/rejected": -0.005706787109375,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.025987150797661156,
|
|
"grad_norm": 1.8960338628946363,
|
|
"learning_rate": 1.2987012987012984e-07,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.679,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": 0.024169921875,
|
|
"rewards/margins": 0.029052734375,
|
|
"rewards/rejected": -0.004791259765625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.02670901609759619,
|
|
"grad_norm": 1.9289106441512014,
|
|
"learning_rate": 1.3347763347763348e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -143.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6755,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 0.0277099609375,
|
|
"rewards/margins": 0.036376953125,
|
|
"rewards/rejected": -0.00848388671875,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.02743088139753122,
|
|
"grad_norm": 2.4318763159683168,
|
|
"learning_rate": 1.370851370851371e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -161.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6738,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": 0.016845703125,
|
|
"rewards/margins": 0.0390625,
|
|
"rewards/rejected": -0.022216796875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.028152746697466254,
|
|
"grad_norm": 1.727518768188907,
|
|
"learning_rate": 1.406926406926407e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6737,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": 0.0238037109375,
|
|
"rewards/margins": 0.04541015625,
|
|
"rewards/rejected": -0.0216064453125,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.028874611997401285,
|
|
"grad_norm": 1.8565593414895172,
|
|
"learning_rate": 1.4430014430014428e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6703,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 0.01324462890625,
|
|
"rewards/margins": 0.0458984375,
|
|
"rewards/rejected": -0.03271484375,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.02959647729733632,
|
|
"grad_norm": 1.9588571879597823,
|
|
"learning_rate": 1.479076479076479e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6681,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 0.006378173828125,
|
|
"rewards/margins": 0.052734375,
|
|
"rewards/rejected": -0.04638671875,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.03031834259727135,
|
|
"grad_norm": 1.93069247030128,
|
|
"learning_rate": 1.5151515151515152e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6667,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -0.0025787353515625,
|
|
"rewards/margins": 0.044189453125,
|
|
"rewards/rejected": -0.046875,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.031040207897206382,
|
|
"grad_norm": 1.819742407006859,
|
|
"learning_rate": 1.5512265512265513e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6625,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.002685546875,
|
|
"rewards/margins": 0.06396484375,
|
|
"rewards/rejected": -0.06640625,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.031762073197141416,
|
|
"grad_norm": 2.3055219231256108,
|
|
"learning_rate": 1.5873015873015872e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6555,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.01123046875,
|
|
"rewards/margins": 0.078125,
|
|
"rewards/rejected": -0.08935546875,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.032483938497076446,
|
|
"grad_norm": 1.9129712763765747,
|
|
"learning_rate": 1.6233766233766232e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6547,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.031494140625,
|
|
"rewards/margins": 0.07666015625,
|
|
"rewards/rejected": -0.10791015625,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.03320580379701148,
|
|
"grad_norm": 1.936887902580248,
|
|
"learning_rate": 1.6594516594516593e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -166.0,
|
|
"logps/rejected": -163.0,
|
|
"loss": 0.65,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.0478515625,
|
|
"rewards/margins": 0.08740234375,
|
|
"rewards/rejected": -0.1357421875,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.03392766909694651,
|
|
"grad_norm": 2.1168551608123725,
|
|
"learning_rate": 1.6955266955266957e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -190.0,
|
|
"loss": 0.6399,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.06298828125,
|
|
"rewards/margins": 0.11279296875,
|
|
"rewards/rejected": -0.17578125,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.034649534396881544,
|
|
"grad_norm": 2.4110075564522533,
|
|
"learning_rate": 1.7316017316017315e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -167.0,
|
|
"loss": 0.6345,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.09375,
|
|
"rewards/margins": 0.1376953125,
|
|
"rewards/rejected": -0.2314453125,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.035371399696816574,
|
|
"grad_norm": 2.561479324237141,
|
|
"learning_rate": 1.7676767676767676e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6272,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.1552734375,
|
|
"rewards/margins": 0.1328125,
|
|
"rewards/rejected": -0.287109375,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.036093264996751605,
|
|
"grad_norm": 2.2009284942320004,
|
|
"learning_rate": 1.8037518037518037e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -180.0,
|
|
"logps/rejected": -198.0,
|
|
"loss": 0.6213,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.1826171875,
|
|
"rewards/margins": 0.1806640625,
|
|
"rewards/rejected": -0.36328125,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.036815130296686635,
|
|
"grad_norm": 2.433624086599741,
|
|
"learning_rate": 1.8398268398268395e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -182.0,
|
|
"logps/rejected": -181.0,
|
|
"loss": 0.6137,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.2294921875,
|
|
"rewards/margins": 0.1787109375,
|
|
"rewards/rejected": -0.408203125,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.03753699559662167,
|
|
"grad_norm": 2.3303081056576396,
|
|
"learning_rate": 1.875901875901876e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -183.0,
|
|
"logps/rejected": -198.0,
|
|
"loss": 0.6089,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.28125,
|
|
"rewards/margins": 0.1953125,
|
|
"rewards/rejected": -0.4765625,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.0382588608965567,
|
|
"grad_norm": 7.529255626517495,
|
|
"learning_rate": 1.911976911976912e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -206.0,
|
|
"loss": 0.6063,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.3046875,
|
|
"rewards/margins": 0.236328125,
|
|
"rewards/rejected": -0.5390625,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.03898072619649173,
|
|
"grad_norm": 2.2072494405878578,
|
|
"learning_rate": 1.948051948051948e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -224.0,
|
|
"loss": 0.5956,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.3515625,
|
|
"rewards/margins": 0.283203125,
|
|
"rewards/rejected": -0.6328125,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.03970259149642677,
|
|
"grad_norm": 2.310476086986689,
|
|
"learning_rate": 1.984126984126984e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -220.0,
|
|
"logps/rejected": -209.0,
|
|
"loss": 0.5867,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.416015625,
|
|
"rewards/margins": 0.236328125,
|
|
"rewards/rejected": -0.65234375,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.0404244567963618,
|
|
"grad_norm": 2.6001658124806286,
|
|
"learning_rate": 2.02020202020202e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -208.0,
|
|
"logps/rejected": -239.0,
|
|
"loss": 0.5754,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.447265625,
|
|
"rewards/margins": 0.31640625,
|
|
"rewards/rejected": -0.76171875,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.04114632209629683,
|
|
"grad_norm": 2.635075269431316,
|
|
"learning_rate": 2.0562770562770563e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -217.0,
|
|
"logps/rejected": -227.0,
|
|
"loss": 0.5719,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.494140625,
|
|
"rewards/margins": 0.33984375,
|
|
"rewards/rejected": -0.83203125,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.04186818739623186,
|
|
"grad_norm": 2.5380650062224066,
|
|
"learning_rate": 2.0923520923520924e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -216.0,
|
|
"logps/rejected": -237.0,
|
|
"loss": 0.5794,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.50390625,
|
|
"rewards/margins": 0.349609375,
|
|
"rewards/rejected": -0.8515625,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.0425900526961669,
|
|
"grad_norm": 2.808866971835476,
|
|
"learning_rate": 2.1284271284271282e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -216.0,
|
|
"logps/rejected": -264.0,
|
|
"loss": 0.5557,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.5390625,
|
|
"rewards/margins": 0.50390625,
|
|
"rewards/rejected": -1.046875,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.04331191799610193,
|
|
"grad_norm": 3.660790591856929,
|
|
"learning_rate": 2.1645021645021643e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -215.0,
|
|
"logps/rejected": -226.0,
|
|
"loss": 0.5374,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.60546875,
|
|
"rewards/margins": 0.3203125,
|
|
"rewards/rejected": -0.92578125,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.04403378329603696,
|
|
"grad_norm": 3.4012692341173842,
|
|
"learning_rate": 2.2005772005772004e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -243.0,
|
|
"logps/rejected": -288.0,
|
|
"loss": 0.5302,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.75,
|
|
"rewards/margins": 0.51953125,
|
|
"rewards/rejected": -1.2734375,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.04475564859597199,
|
|
"grad_norm": 3.1213723367337276,
|
|
"learning_rate": 2.2366522366522368e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -252.0,
|
|
"logps/rejected": -290.0,
|
|
"loss": 0.5349,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.82421875,
|
|
"rewards/margins": 0.6171875,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.045477513895907026,
|
|
"grad_norm": 3.4139219834989825,
|
|
"learning_rate": 2.2727272727272726e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -241.0,
|
|
"logps/rejected": -290.0,
|
|
"loss": 0.5313,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.76953125,
|
|
"rewards/margins": 0.625,
|
|
"rewards/rejected": -1.390625,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.04619937919584206,
|
|
"grad_norm": 3.545601457816913,
|
|
"learning_rate": 2.3088023088023087e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -254.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5163,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.796875,
|
|
"rewards/margins": 0.671875,
|
|
"rewards/rejected": -1.46875,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.04692124449577709,
|
|
"grad_norm": 6.320535114176617,
|
|
"learning_rate": 2.3448773448773448e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -258.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.5168,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -0.92578125,
|
|
"rewards/margins": 0.625,
|
|
"rewards/rejected": -1.5546875,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.04764310979571212,
|
|
"grad_norm": 4.40606882426551,
|
|
"learning_rate": 2.3809523809523806e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -258.0,
|
|
"logps/rejected": -300.0,
|
|
"loss": 0.506,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.9375,
|
|
"rewards/margins": 0.6171875,
|
|
"rewards/rejected": -1.5546875,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.048364975095647154,
|
|
"grad_norm": 7.228420353041346,
|
|
"learning_rate": 2.4170274170274167e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -230.0,
|
|
"logps/rejected": -308.0,
|
|
"loss": 0.5231,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.8359375,
|
|
"rewards/margins": 0.7421875,
|
|
"rewards/rejected": -1.578125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.049086840395582185,
|
|
"grad_norm": 4.810449590822843,
|
|
"learning_rate": 2.453102453102453e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -246.0,
|
|
"logps/rejected": -280.0,
|
|
"loss": 0.5107,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.88671875,
|
|
"rewards/margins": 0.498046875,
|
|
"rewards/rejected": -1.3828125,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.049808705695517215,
|
|
"grad_norm": 6.916956622472092,
|
|
"learning_rate": 2.4891774891774894e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -270.0,
|
|
"logps/rejected": -322.0,
|
|
"loss": 0.5105,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.9765625,
|
|
"rewards/margins": 0.6484375,
|
|
"rewards/rejected": -1.625,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.050530570995452245,
|
|
"grad_norm": 5.072218871042774,
|
|
"learning_rate": 2.525252525252525e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -256.0,
|
|
"logps/rejected": -326.0,
|
|
"loss": 0.5065,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.92578125,
|
|
"rewards/margins": 0.76953125,
|
|
"rewards/rejected": -1.6953125,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.05125243629538728,
|
|
"grad_norm": 8.875662849487487,
|
|
"learning_rate": 2.5613275613275616e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -254.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5112,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.94921875,
|
|
"rewards/margins": 0.61328125,
|
|
"rewards/rejected": -1.5625,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.05197430159532231,
|
|
"grad_norm": 5.595849818392523,
|
|
"learning_rate": 2.597402597402597e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -264.0,
|
|
"logps/rejected": -340.0,
|
|
"loss": 0.5107,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.0078125,
|
|
"rewards/margins": 0.8125,
|
|
"rewards/rejected": -1.8203125,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.05269616689525734,
|
|
"grad_norm": 5.404147285645744,
|
|
"learning_rate": 2.633477633477633e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -368.0,
|
|
"loss": 0.5035,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.984375,
|
|
"rewards/margins": 0.94140625,
|
|
"rewards/rejected": -1.9296875,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.05341803219519238,
|
|
"grad_norm": 8.76041109611346,
|
|
"learning_rate": 2.6695526695526696e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -342.0,
|
|
"loss": 0.4887,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -1.125,
|
|
"rewards/margins": 0.75390625,
|
|
"rewards/rejected": -1.8828125,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.05413989749512741,
|
|
"grad_norm": 7.025589443972428,
|
|
"learning_rate": 2.7056277056277054e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -262.0,
|
|
"logps/rejected": -348.0,
|
|
"loss": 0.4823,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.0,
|
|
"rewards/margins": 0.94921875,
|
|
"rewards/rejected": -1.953125,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.05486176279506244,
|
|
"grad_norm": 11.975302977639574,
|
|
"learning_rate": 2.741702741702742e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -322.0,
|
|
"loss": 0.4766,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.140625,
|
|
"rewards/margins": 0.6953125,
|
|
"rewards/rejected": -1.8359375,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.05558362809499747,
|
|
"grad_norm": 6.5118457359444,
|
|
"learning_rate": 2.7777777777777776e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -346.0,
|
|
"loss": 0.4846,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.1875,
|
|
"rewards/margins": 0.80859375,
|
|
"rewards/rejected": -1.9921875,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.05630549339493251,
|
|
"grad_norm": 6.106781577833728,
|
|
"learning_rate": 2.813852813852814e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -260.0,
|
|
"logps/rejected": -356.0,
|
|
"loss": 0.4665,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.046875,
|
|
"rewards/margins": 0.83203125,
|
|
"rewards/rejected": -1.875,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.05702735869486754,
|
|
"grad_norm": 10.75495308386994,
|
|
"learning_rate": 2.8499278499278503e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -284.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4695,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.2109375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.390625,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.05774922399480257,
|
|
"grad_norm": 5.788421207967755,
|
|
"learning_rate": 2.8860028860028856e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.4767,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.125,
|
|
"rewards/margins": 0.84765625,
|
|
"rewards/rejected": -1.9765625,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.0584710892947376,
|
|
"grad_norm": 8.78614857490401,
|
|
"learning_rate": 2.922077922077922e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -262.0,
|
|
"logps/rejected": -362.0,
|
|
"loss": 0.4722,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -2.125,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.05919295459467264,
|
|
"grad_norm": 7.3161771388355294,
|
|
"learning_rate": 2.958152958152958e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.4996,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 0.93359375,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.05991481989460767,
|
|
"grad_norm": 8.266553993044619,
|
|
"learning_rate": 2.994227994227994e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -376.0,
|
|
"loss": 0.4545,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 1.015625,
|
|
"rewards/rejected": -2.171875,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.0606366851945427,
|
|
"grad_norm": 8.218438777679667,
|
|
"learning_rate": 3.0303030303030305e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -382.0,
|
|
"loss": 0.4639,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 1.0234375,
|
|
"rewards/rejected": -2.28125,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.06135855049447773,
|
|
"grad_norm": 8.61378341353772,
|
|
"learning_rate": 3.0663780663780663e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -284.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4536,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.2578125,
|
|
"rewards/margins": 1.109375,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.062080415794412765,
|
|
"grad_norm": 12.398406848670698,
|
|
"learning_rate": 3.1024531024531027e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.476,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.2578125,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -2.265625,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.0628022810943478,
|
|
"grad_norm": 12.244121744087602,
|
|
"learning_rate": 3.138528138528138e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -388.0,
|
|
"loss": 0.4623,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 0.984375,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.06352414639428283,
|
|
"grad_norm": 10.812869810884447,
|
|
"learning_rate": 3.1746031746031743e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -366.0,
|
|
"loss": 0.4498,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.234375,
|
|
"rewards/margins": 0.99609375,
|
|
"rewards/rejected": -2.234375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.06424601169421786,
|
|
"grad_norm": 11.229212598361977,
|
|
"learning_rate": 3.2106782106782107e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4373,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.3828125,
|
|
"rewards/margins": 1.0859375,
|
|
"rewards/rejected": -2.46875,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.06496787699415289,
|
|
"grad_norm": 7.414765021326351,
|
|
"learning_rate": 3.2467532467532465e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4253,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.140625,
|
|
"rewards/rejected": -2.484375,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.06568974229408793,
|
|
"grad_norm": 6.656174306105041,
|
|
"learning_rate": 3.282828282828283e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -440.0,
|
|
"loss": 0.4366,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.06641160759402295,
|
|
"grad_norm": 6.986955189515223,
|
|
"learning_rate": 3.3189033189033187e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -422.0,
|
|
"loss": 0.4335,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.234375,
|
|
"rewards/rejected": -2.75,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.06713347289395799,
|
|
"grad_norm": 8.704691417598838,
|
|
"learning_rate": 3.354978354978355e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.4048,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.0703125,
|
|
"rewards/rejected": -2.640625,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.06785533819389301,
|
|
"grad_norm": 19.52535270358171,
|
|
"learning_rate": 3.3910533910533914e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.434,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.6015625,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.06857720349382805,
|
|
"grad_norm": 11.339737953685198,
|
|
"learning_rate": 3.4271284271284267e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -398.0,
|
|
"loss": 0.4334,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.5625,
|
|
"rewards/margins": 1.03125,
|
|
"rewards/rejected": -2.59375,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.06929906879376309,
|
|
"grad_norm": 10.31037341226185,
|
|
"learning_rate": 3.463203463203463e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4161,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -3.046875,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.07002093409369811,
|
|
"grad_norm": 11.130584733664083,
|
|
"learning_rate": 3.499278499278499e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -424.0,
|
|
"loss": 0.4078,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.4296875,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -2.796875,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.07074279939363315,
|
|
"grad_norm": 9.06083640139615,
|
|
"learning_rate": 3.535353535353535e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -442.0,
|
|
"loss": 0.4178,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -2.890625,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.07146466469356819,
|
|
"grad_norm": 9.541422651232944,
|
|
"learning_rate": 3.5714285714285716e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.3953,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.375,
|
|
"rewards/rejected": -3.078125,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.07218652999350321,
|
|
"grad_norm": 9.110551563711448,
|
|
"learning_rate": 3.6075036075036074e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -464.0,
|
|
"loss": 0.4021,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -2.984375,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.07290839529343825,
|
|
"grad_norm": 15.071837561832831,
|
|
"learning_rate": 3.643578643578644e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.4045,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5625,
|
|
"rewards/margins": 1.3671875,
|
|
"rewards/rejected": -2.921875,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.07363026059337327,
|
|
"grad_norm": 13.920977378141336,
|
|
"learning_rate": 3.679653679653679e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.431,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.8125,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -3.28125,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.07435212589330831,
|
|
"grad_norm": 11.085289038392293,
|
|
"learning_rate": 3.7157287157287154e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.3934,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.390625,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.07507399119324334,
|
|
"grad_norm": 10.088971897286424,
|
|
"learning_rate": 3.751803751803752e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4073,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.07579585649317837,
|
|
"grad_norm": 43.24163673888106,
|
|
"learning_rate": 3.7878787878787876e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -496.0,
|
|
"loss": 0.4152,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.890625,
|
|
"rewards/margins": 1.25,
|
|
"rewards/rejected": -3.140625,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.0765177217931134,
|
|
"grad_norm": 8.537892452168746,
|
|
"learning_rate": 3.823953823953824e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -480.0,
|
|
"loss": 0.3947,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.765625,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.07723958709304844,
|
|
"grad_norm": 9.485771716655114,
|
|
"learning_rate": 3.86002886002886e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.3837,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.453125,
|
|
"rewards/rejected": -3.09375,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.07796145239298347,
|
|
"grad_norm": 9.46365785011617,
|
|
"learning_rate": 3.896103896103896e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.3848,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.15625,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.9375,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.0786833176929185,
|
|
"grad_norm": 13.461346342787172,
|
|
"learning_rate": 3.9321789321789325e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -360.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3922,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -3.765625,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.07940518299285354,
|
|
"grad_norm": 12.44634151581957,
|
|
"learning_rate": 3.968253968253968e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -508.0,
|
|
"loss": 0.3727,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -2.015625,
|
|
"rewards/margins": 1.5,
|
|
"rewards/rejected": -3.515625,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.08012704829278856,
|
|
"grad_norm": 11.687822831294243,
|
|
"learning_rate": 4.004329004329004e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -568.0,
|
|
"loss": 0.3765,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.0625,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.0808489135927236,
|
|
"grad_norm": 16.193115895860167,
|
|
"learning_rate": 4.04040404040404e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -532.0,
|
|
"loss": 0.3726,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.9921875,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.08157077889265862,
|
|
"grad_norm": 10.454280347519187,
|
|
"learning_rate": 4.0764790764790763e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -350.0,
|
|
"logps/rejected": -496.0,
|
|
"loss": 0.4066,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.78125,
|
|
"rewards/margins": 1.515625,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.08229264419259366,
|
|
"grad_norm": 10.033847939954763,
|
|
"learning_rate": 4.1125541125541127e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.355,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.7890625,
|
|
"rewards/margins": 1.3984375,
|
|
"rewards/rejected": -3.1875,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.0830145094925287,
|
|
"grad_norm": 13.538065077526934,
|
|
"learning_rate": 4.1486291486291485e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -398.0,
|
|
"logps/rejected": -588.0,
|
|
"loss": 0.3843,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.1875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.08373637479246372,
|
|
"grad_norm": 12.06165274825125,
|
|
"learning_rate": 4.184704184704185e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -500.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.6796875,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.08445824009239876,
|
|
"grad_norm": 14.245004620424362,
|
|
"learning_rate": 4.22077922077922e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -376.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3599,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.21875,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.0851801053923338,
|
|
"grad_norm": 12.93467746542696,
|
|
"learning_rate": 4.2568542568542565e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -330.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.3635,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.75,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -3.34375,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.08590197069226882,
|
|
"grad_norm": 11.202101570401776,
|
|
"learning_rate": 4.292929292929293e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.3583,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.125,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -3.953125,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.08662383599220386,
|
|
"grad_norm": 10.771994466129188,
|
|
"learning_rate": 4.3290043290043287e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -358.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.3465,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.8828125,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.08734570129213888,
|
|
"grad_norm": 14.841268368598595,
|
|
"learning_rate": 4.365079365079365e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3477,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.08806756659207392,
|
|
"grad_norm": 9.504108427187644,
|
|
"learning_rate": 4.401154401154401e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -394.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3755,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.25,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.08878943189200895,
|
|
"grad_norm": 12.115514229718103,
|
|
"learning_rate": 4.437229437229437e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -304.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3704,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -1.5390625,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -3.71875,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.08951129719194398,
|
|
"grad_norm": 10.317228267259349,
|
|
"learning_rate": 4.4733044733044736e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3559,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.09023316249187902,
|
|
"grad_norm": 11.197522859178363,
|
|
"learning_rate": 4.509379509379509e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3219,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.859375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.09095502779181405,
|
|
"grad_norm": 13.497331724381818,
|
|
"learning_rate": 4.545454545454545e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3388,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.546875,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.09167689309174908,
|
|
"grad_norm": 11.433124284340494,
|
|
"learning_rate": 4.581529581529581e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.3458,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -4.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.09239875839168411,
|
|
"grad_norm": 12.324384516756012,
|
|
"learning_rate": 4.6176046176046174e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -410.0,
|
|
"logps/rejected": -580.0,
|
|
"loss": 0.3426,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.09312062369161915,
|
|
"grad_norm": 10.730987022263216,
|
|
"learning_rate": 4.6536796536796537e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3593,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -4.625,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.09384248899155417,
|
|
"grad_norm": 10.132938191746307,
|
|
"learning_rate": 4.6897546897546896e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -408.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3632,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.359375,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -4.09375,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.09456435429148921,
|
|
"grad_norm": 11.988621181927288,
|
|
"learning_rate": 4.725829725829726e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.3609,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.921875,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.09528621959142423,
|
|
"grad_norm": 8.802084815882347,
|
|
"learning_rate": 4.761904761904761e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3197,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.9375,
|
|
"rewards/margins": 1.5859375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.09600808489135927,
|
|
"grad_norm": 13.978035740824382,
|
|
"learning_rate": 4.797979797979798e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.355,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.09672995019129431,
|
|
"grad_norm": 9.329953252240916,
|
|
"learning_rate": 4.834054834054833e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3382,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.09745181549122933,
|
|
"grad_norm": 14.354032725201476,
|
|
"learning_rate": 4.87012987012987e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3375,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.53125,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.09817368079116437,
|
|
"grad_norm": 11.184165035806638,
|
|
"learning_rate": 4.906204906204906e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -640.0,
|
|
"loss": 0.3528,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.0988955460910994,
|
|
"grad_norm": 9.32730305340813,
|
|
"learning_rate": 4.942279942279942e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.3343,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 1.9375,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.09961741139103443,
|
|
"grad_norm": 17.291834755334516,
|
|
"learning_rate": 4.978354978354979e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3246,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.9765625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.10033927669096947,
|
|
"grad_norm": 10.743034154219572,
|
|
"learning_rate": 4.999998729993963e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3335,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.10106114199090449,
|
|
"grad_norm": 17.190340388139088,
|
|
"learning_rate": 4.999984442440868e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3204,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.10178300729083953,
|
|
"grad_norm": 9.641498367561857,
|
|
"learning_rate": 4.99995427991816e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3379,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.10250487259077457,
|
|
"grad_norm": 9.768069678095703,
|
|
"learning_rate": 4.999908242617371e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.34,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.10322673789070959,
|
|
"grad_norm": 13.37422488078675,
|
|
"learning_rate": 4.99984633083084e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.331,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 1.8984375,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.10394860319064463,
|
|
"grad_norm": 11.253293027609532,
|
|
"learning_rate": 4.99976854495171e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.3218,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.453125,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.10467046849057966,
|
|
"grad_norm": 10.82613372416563,
|
|
"learning_rate": 4.999674885473922e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3213,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.921875,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -4.65625,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.10539233379051469,
|
|
"grad_norm": 15.794446739298891,
|
|
"learning_rate": 4.999565352992216e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.3289,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.10611419909044972,
|
|
"grad_norm": 10.361664841175285,
|
|
"learning_rate": 4.999439948202127e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.3261,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.10683606439038476,
|
|
"grad_norm": 10.555951221100015,
|
|
"learning_rate": 4.999298671899977e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3399,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.10755792969031978,
|
|
"grad_norm": 9.802204192641314,
|
|
"learning_rate": 4.999141524982877e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.309,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.10827979499025482,
|
|
"grad_norm": 12.72303480502336,
|
|
"learning_rate": 4.998968508448714e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -632.0,
|
|
"loss": 0.3276,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -4.78125,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.10900166029018984,
|
|
"grad_norm": 14.570251314538908,
|
|
"learning_rate": 4.998779623396146e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.2995,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.10972352559012488,
|
|
"grad_norm": 11.169758259343533,
|
|
"learning_rate": 4.9985748710246e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3344,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.11044539089005992,
|
|
"grad_norm": 10.504554948014457,
|
|
"learning_rate": 4.998354252634257e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -486.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.2853,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.11116725618999494,
|
|
"grad_norm": 15.073212833972551,
|
|
"learning_rate": 4.998117769626051e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.309,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.11188912148992998,
|
|
"grad_norm": 10.298569511142073,
|
|
"learning_rate": 4.997865423501657e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -380.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3261,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.11261098678986502,
|
|
"grad_norm": 11.022262328486809,
|
|
"learning_rate": 4.997597215863477e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2878,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.11333285208980004,
|
|
"grad_norm": 12.425423187064853,
|
|
"learning_rate": 4.99731314841464e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.3008,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.11405471738973508,
|
|
"grad_norm": 10.996930809431353,
|
|
"learning_rate": 4.997013222958978e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.3105,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.1147765826896701,
|
|
"grad_norm": 11.11449736666811,
|
|
"learning_rate": 4.99669744140103e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.3043,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.11549844798960514,
|
|
"grad_norm": 10.155961204890554,
|
|
"learning_rate": 4.996365805746015e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -474.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2908,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.11622031328954018,
|
|
"grad_norm": 9.079844890608808,
|
|
"learning_rate": 4.996018318099829e-07,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2977,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.1169421785894752,
|
|
"grad_norm": 13.337729397169802,
|
|
"learning_rate": 4.995654980669028e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2937,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.03125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.11766404388941024,
|
|
"grad_norm": 9.987891055540013,
|
|
"learning_rate": 4.995275795760816e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.2991,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.11838590918934527,
|
|
"grad_norm": 12.804317711855225,
|
|
"learning_rate": 4.994880765783026e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2883,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.1191077744892803,
|
|
"grad_norm": 11.589955875632223,
|
|
"learning_rate": 4.99446989324411e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2847,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.11982963978921533,
|
|
"grad_norm": 14.992092750756044,
|
|
"learning_rate": 4.99404318075312e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2926,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.12055150508915037,
|
|
"grad_norm": 14.63652475825442,
|
|
"learning_rate": 4.993600631019689e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3115,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.1212733703890854,
|
|
"grad_norm": 10.01896442134856,
|
|
"learning_rate": 4.993142246854024e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2917,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.12199523568902043,
|
|
"grad_norm": 10.480137264657502,
|
|
"learning_rate": 4.992668031166876e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.3114,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.12271710098895545,
|
|
"grad_norm": 9.543166172183264,
|
|
"learning_rate": 4.992177986969526e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.3235,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.12343896628889049,
|
|
"grad_norm": 9.39264085167921,
|
|
"learning_rate": 4.991672117373769e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2971,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.12416083158882553,
|
|
"grad_norm": 9.35509206624716,
|
|
"learning_rate": 4.991150425591891e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.2908,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.12488269688876055,
|
|
"grad_norm": 9.262201280713281,
|
|
"learning_rate": 4.99061291493665e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2925,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.1256045621886956,
|
|
"grad_norm": 10.008148122591312,
|
|
"learning_rate": 4.99005958882125e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.3133,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.12632642748863063,
|
|
"grad_norm": 8.554136981806407,
|
|
"learning_rate": 4.989490450759331e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2649,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.12704829278856566,
|
|
"grad_norm": 8.929491173417231,
|
|
"learning_rate": 4.988905504364933e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2715,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.12777015808850067,
|
|
"grad_norm": 19.85583075822656,
|
|
"learning_rate": 4.988304753352482e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.046875,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.1284920233884357,
|
|
"grad_norm": 11.573857511888404,
|
|
"learning_rate": 4.987688201536764e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3207,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.12921388868837075,
|
|
"grad_norm": 10.767926076980798,
|
|
"learning_rate": 4.987055852832899e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2768,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.296875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.12993575398830579,
|
|
"grad_norm": 9.480524665881354,
|
|
"learning_rate": 4.986407711256319e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2996,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.13065761928824082,
|
|
"grad_norm": 11.059595953397832,
|
|
"learning_rate": 4.98574378092274e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2706,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.13137948458817586,
|
|
"grad_norm": 12.559652259820835,
|
|
"learning_rate": 4.985064066048139e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.3016,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.13210134988811087,
|
|
"grad_norm": 16.780300866871737,
|
|
"learning_rate": 4.984368570948724e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.2943,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.1328232151880459,
|
|
"grad_norm": 10.65248837076819,
|
|
"learning_rate": 4.983657300040907e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.2699,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.13354508048798094,
|
|
"grad_norm": 10.948003648298677,
|
|
"learning_rate": 4.982930257841278e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2941,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.13426694578791598,
|
|
"grad_norm": 12.047287233162004,
|
|
"learning_rate": 4.982187448966575e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2898,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.640625,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.13498881108785102,
|
|
"grad_norm": 14.9941388186048,
|
|
"learning_rate": 4.981428878133656e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2811,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.13571067638778603,
|
|
"grad_norm": 9.951835424657778,
|
|
"learning_rate": 4.980654550159463e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.2732,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.13643254168772107,
|
|
"grad_norm": 8.861361984310934,
|
|
"learning_rate": 4.979864469961004e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2762,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.1371544069876561,
|
|
"grad_norm": 9.692735634395657,
|
|
"learning_rate": 4.979058642555307e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2931,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.13787627228759114,
|
|
"grad_norm": 9.565594633496087,
|
|
"learning_rate": 4.978237073059399e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2898,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.13859813758752618,
|
|
"grad_norm": 11.007909649107406,
|
|
"learning_rate": 4.977399766690269e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2902,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.1393200028874612,
|
|
"grad_norm": 9.107456714393756,
|
|
"learning_rate": 4.976546728764836e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.284,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.14004186818739622,
|
|
"grad_norm": 8.927298343498093,
|
|
"learning_rate": 4.975677964699912e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2524,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.14076373348733126,
|
|
"grad_norm": 10.144209347121677,
|
|
"learning_rate": 4.974793480012174e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2621,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.1414855987872663,
|
|
"grad_norm": 7.533895015502329,
|
|
"learning_rate": 4.973893280318125e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2844,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.14220746408720134,
|
|
"grad_norm": 13.017212573238576,
|
|
"learning_rate": 4.972977371334056e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.2873,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.14292932938713637,
|
|
"grad_norm": 8.958119059657708,
|
|
"learning_rate": 4.972045758876014e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2477,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.14365119468707138,
|
|
"grad_norm": 10.493203877125167,
|
|
"learning_rate": 4.971098448859766e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.2933,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.14437305998700642,
|
|
"grad_norm": 9.973862051012356,
|
|
"learning_rate": 4.970135447300752e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2595,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.14509492528694146,
|
|
"grad_norm": 13.28264552294119,
|
|
"learning_rate": 4.969156760314064e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2904,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.640625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.1458167905868765,
|
|
"grad_norm": 8.872169286182675,
|
|
"learning_rate": 4.968162394114387e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2523,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.14653865588681153,
|
|
"grad_norm": 12.757195149734072,
|
|
"learning_rate": 4.967152355015974e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2783,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.14726052118674654,
|
|
"grad_norm": 12.796953074989913,
|
|
"learning_rate": 4.966126649432603e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2723,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.53125,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.14798238648668158,
|
|
"grad_norm": 19.108616982264344,
|
|
"learning_rate": 4.96508528387753e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2763,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.14870425178661661,
|
|
"grad_norm": 9.380783201026548,
|
|
"learning_rate": 4.964028264963456e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.14942611708655165,
|
|
"grad_norm": 10.81595001993398,
|
|
"learning_rate": 4.962955599402481e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.266,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -6.25,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.1501479823864867,
|
|
"grad_norm": 11.456291625001526,
|
|
"learning_rate": 4.961867294006059e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.3064,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.15086984768642173,
|
|
"grad_norm": 7.960182686516041,
|
|
"learning_rate": 4.96076335568496e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2467,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.15159171298635674,
|
|
"grad_norm": 10.25400623338689,
|
|
"learning_rate": 4.959643791449222e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2977,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.375,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.15231357828629177,
|
|
"grad_norm": 10.09340211565806,
|
|
"learning_rate": 4.958508608408109e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2829,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.1530354435862268,
|
|
"grad_norm": 8.537784621679538,
|
|
"learning_rate": 4.957357813770064e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.15375730888616185,
|
|
"grad_norm": 8.985222338453365,
|
|
"learning_rate": 4.956191414842665e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2629,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.15447917418609688,
|
|
"grad_norm": 10.993021927824719,
|
|
"learning_rate": 4.955009419032575e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2681,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.1552010394860319,
|
|
"grad_norm": 11.792550816411442,
|
|
"learning_rate": 4.953811833845503e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.265,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.15592290478596693,
|
|
"grad_norm": 13.163433586155344,
|
|
"learning_rate": 4.952598666886145e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2516,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.15664477008590197,
|
|
"grad_norm": 15.36607001210862,
|
|
"learning_rate": 4.951369925858147e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.2598,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.157366635385837,
|
|
"grad_norm": 8.701540096554936,
|
|
"learning_rate": 4.950125618564046e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.251,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.625,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.15808850068577204,
|
|
"grad_norm": 10.616304811087307,
|
|
"learning_rate": 4.948865752905228e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2533,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.15881036598570708,
|
|
"grad_norm": 10.818629515177625,
|
|
"learning_rate": 4.947590336881874e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2593,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.1595322312856421,
|
|
"grad_norm": 10.595421594796772,
|
|
"learning_rate": 4.946299378592912e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2795,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.578125,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.16025409658557713,
|
|
"grad_norm": 13.24386115549115,
|
|
"learning_rate": 4.944992886235961e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2521,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.16097596188551216,
|
|
"grad_norm": 8.619302388111215,
|
|
"learning_rate": 4.943670868107284e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.27,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.1616978271854472,
|
|
"grad_norm": 8.272453046055942,
|
|
"learning_rate": 4.942333332601731e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.16241969248538224,
|
|
"grad_norm": 6.171166558818759,
|
|
"learning_rate": 4.940980288212691e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2574,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.16314155778531725,
|
|
"grad_norm": 9.42178572046336,
|
|
"learning_rate": 4.939611743532031e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2385,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.16386342308525229,
|
|
"grad_norm": 8.438515261081413,
|
|
"learning_rate": 4.93822770725005e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2737,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.16458528838518732,
|
|
"grad_norm": 11.043392051990788,
|
|
"learning_rate": 4.936828188155413e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.625,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.16530715368512236,
|
|
"grad_norm": 10.831257780639715,
|
|
"learning_rate": 4.935413195135106e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.1660290189850574,
|
|
"grad_norm": 9.747157511340994,
|
|
"learning_rate": 4.933982737174374e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2701,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.1667508842849924,
|
|
"grad_norm": 9.870871070277088,
|
|
"learning_rate": 4.932536823356664e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2543,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.16747274958492744,
|
|
"grad_norm": 8.8575895636107,
|
|
"learning_rate": 4.931075462863567e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2594,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.16819461488486248,
|
|
"grad_norm": 7.816395810688846,
|
|
"learning_rate": 4.929598664974762e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.16891648018479752,
|
|
"grad_norm": 8.344064880799563,
|
|
"learning_rate": 4.928106439067958e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2669,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.16963834548473256,
|
|
"grad_norm": 10.994252200422187,
|
|
"learning_rate": 4.926598794618829e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.278,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.1703602107846676,
|
|
"grad_norm": 15.14815293816792,
|
|
"learning_rate": 4.92507574120096e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2777,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.1710820760846026,
|
|
"grad_norm": 7.821370479620557,
|
|
"learning_rate": 4.923537288485779e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2638,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.17180394138453764,
|
|
"grad_norm": 9.211826498394792,
|
|
"learning_rate": 4.921983446242506e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2493,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.17252580668447268,
|
|
"grad_norm": 8.298256232187494,
|
|
"learning_rate": 4.92041422433808e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2348,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.17324767198440771,
|
|
"grad_norm": 10.983625039262604,
|
|
"learning_rate": 4.918829632737103e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2633,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.17396953728434275,
|
|
"grad_norm": 12.61039915689691,
|
|
"learning_rate": 4.917229681501774e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.264,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.17469140258427776,
|
|
"grad_norm": 11.643682996836732,
|
|
"learning_rate": 4.91561438079183e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2668,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.1754132678842128,
|
|
"grad_norm": 13.088952482660957,
|
|
"learning_rate": 4.913983740864473e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2359,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.17613513318414784,
|
|
"grad_norm": 10.188119267424455,
|
|
"learning_rate": 4.91233777207431e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2748,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.17685699848408287,
|
|
"grad_norm": 8.609826083562075,
|
|
"learning_rate": 4.910676484873292e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2658,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.1775788637840179,
|
|
"grad_norm": 10.48253643988837,
|
|
"learning_rate": 4.908999889810633e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2331,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.17830072908395295,
|
|
"grad_norm": 17.344352556263694,
|
|
"learning_rate": 4.907307997532761e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2354,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.17902259438388796,
|
|
"grad_norm": 9.982854645114383,
|
|
"learning_rate": 4.905600818783237e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.3273,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.179744459683823,
|
|
"grad_norm": 10.68094022282627,
|
|
"learning_rate": 4.903878364402691e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2468,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.18046632498375803,
|
|
"grad_norm": 8.296383042780498,
|
|
"learning_rate": 4.902140645328759e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2599,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.18118819028369307,
|
|
"grad_norm": 9.103787824302069,
|
|
"learning_rate": 4.900387672596003e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2318,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.1819100555836281,
|
|
"grad_norm": 16.67988360600736,
|
|
"learning_rate": 4.898619457335848e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.248,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.18263192088356311,
|
|
"grad_norm": 10.469848545031832,
|
|
"learning_rate": 4.896836010776509e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2595,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.18335378618349815,
|
|
"grad_norm": 9.661637065408652,
|
|
"learning_rate": 4.895037344242921e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2585,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.09375,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.1840756514834332,
|
|
"grad_norm": 8.853616581850865,
|
|
"learning_rate": 4.893223469156664e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.249,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.18479751678336823,
|
|
"grad_norm": 14.23122096213618,
|
|
"learning_rate": 4.891394397035896e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2458,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.18551938208330326,
|
|
"grad_norm": 7.165047283079565,
|
|
"learning_rate": 4.889550139495275e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2193,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.1862412473832383,
|
|
"grad_norm": 9.892022723104382,
|
|
"learning_rate": 4.887690708245887e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2276,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.765625,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.1869631126831733,
|
|
"grad_norm": 9.543867068973059,
|
|
"learning_rate": 4.885816115095171e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2391,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.18768497798310835,
|
|
"grad_norm": 10.889738728754864,
|
|
"learning_rate": 4.883926371946843e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2702,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.18840684328304338,
|
|
"grad_norm": 6.728294158129367,
|
|
"learning_rate": 4.882021490800826e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.18912870858297842,
|
|
"grad_norm": 9.374612134195704,
|
|
"learning_rate": 4.880101483753167e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2198,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.18985057388291346,
|
|
"grad_norm": 8.403906697614635,
|
|
"learning_rate": 4.878166362995963e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2728,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.19057243918284847,
|
|
"grad_norm": 9.739452944604345,
|
|
"learning_rate": 4.876216140817285e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2399,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.1912943044827835,
|
|
"grad_norm": 9.412524327526398,
|
|
"learning_rate": 4.874250829601094e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.19201616978271854,
|
|
"grad_norm": 9.67143825795057,
|
|
"learning_rate": 4.872270441827174e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2358,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.19273803508265358,
|
|
"grad_norm": 10.225198499138108,
|
|
"learning_rate": 4.870274990071038e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.19345990038258862,
|
|
"grad_norm": 8.93422692887298,
|
|
"learning_rate": 4.868264487003862e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2518,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.19418176568252365,
|
|
"grad_norm": 8.108160852039612,
|
|
"learning_rate": 4.866238945392393e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2465,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.19490363098245866,
|
|
"grad_norm": 8.305890952481619,
|
|
"learning_rate": 4.864198378098877e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.237,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.1956254962823937,
|
|
"grad_norm": 12.954558843929119,
|
|
"learning_rate": 4.862142798080973e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2245,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.19634736158232874,
|
|
"grad_norm": 6.775612276163364,
|
|
"learning_rate": 4.860072218391669e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2153,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.19706922688226378,
|
|
"grad_norm": 8.770593576643066,
|
|
"learning_rate": 4.857986652179203e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2534,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.1977910921821988,
|
|
"grad_norm": 8.729235176349604,
|
|
"learning_rate": 4.855886112686977e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.19851295748213382,
|
|
"grad_norm": 10.466383153721448,
|
|
"learning_rate": 4.853770613253476e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.19923482278206886,
|
|
"grad_norm": 10.738593768361735,
|
|
"learning_rate": 4.851640167312178e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.224,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.1999566880820039,
|
|
"grad_norm": 9.416300695572696,
|
|
"learning_rate": 4.849494788391473e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2284,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.20067855338193893,
|
|
"grad_norm": 9.528669823636646,
|
|
"learning_rate": 4.847334490114576e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2293,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.20140041868187397,
|
|
"grad_norm": 9.797059355953671,
|
|
"learning_rate": 4.84515928619944e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2094,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.20212228398180898,
|
|
"grad_norm": 9.406533092835442,
|
|
"learning_rate": 4.84296919045867e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2334,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.20284414928174402,
|
|
"grad_norm": 9.254549987899654,
|
|
"learning_rate": 4.840764216799433e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2337,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.20356601458167906,
|
|
"grad_norm": 9.14538857003236,
|
|
"learning_rate": 4.838544379223373e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2619,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.2042878798816141,
|
|
"grad_norm": 9.535277544276601,
|
|
"learning_rate": 4.836309691826518e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.238,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.20500974518154913,
|
|
"grad_norm": 8.327834581371992,
|
|
"learning_rate": 4.834060168799195e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2305,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.20573161048148417,
|
|
"grad_norm": 7.7347670431429405,
|
|
"learning_rate": 4.831795824425938e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2542,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.20645347578141918,
|
|
"grad_norm": 8.15521635648461,
|
|
"learning_rate": 4.829516673085394e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.20717534108135421,
|
|
"grad_norm": 8.022215432273788,
|
|
"learning_rate": 4.827222729250236e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2461,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.20789720638128925,
|
|
"grad_norm": 7.199943747449723,
|
|
"learning_rate": 4.824914007487072e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2155,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.2086190716812243,
|
|
"grad_norm": 12.583865202253415,
|
|
"learning_rate": 4.822590522456347e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2309,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.20934093698115933,
|
|
"grad_norm": 10.061884816438084,
|
|
"learning_rate": 4.820252288912254e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.21006280228109434,
|
|
"grad_norm": 8.24310045419468,
|
|
"learning_rate": 4.817899321702642e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2341,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.21078466758102937,
|
|
"grad_norm": 9.993736684687924,
|
|
"learning_rate": 4.815531635768916e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2298,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.2115065328809644,
|
|
"grad_norm": 8.215550634524817,
|
|
"learning_rate": 4.813149246145946e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2413,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.21222839818089945,
|
|
"grad_norm": 6.880787558392017,
|
|
"learning_rate": 4.810752167961971e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2364,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.21295026348083448,
|
|
"grad_norm": 8.022287212940531,
|
|
"learning_rate": 4.808340416438505e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2212,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.21367212878076952,
|
|
"grad_norm": 10.935811215207599,
|
|
"learning_rate": 4.805914006890234e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2454,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.21439399408070453,
|
|
"grad_norm": 7.928137164090748,
|
|
"learning_rate": 4.803472954724928e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2296,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.21511585938063957,
|
|
"grad_norm": 10.309732813070605,
|
|
"learning_rate": 4.801017275443331e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2371,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.2158377246805746,
|
|
"grad_norm": 9.479637952138695,
|
|
"learning_rate": 4.798546984639076e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.21655958998050964,
|
|
"grad_norm": 11.653196066334537,
|
|
"learning_rate": 4.796062097998578e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.21728145528044468,
|
|
"grad_norm": 12.258514957096317,
|
|
"learning_rate": 4.793562631300932e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2292,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.2180033205803797,
|
|
"grad_norm": 7.907531921224019,
|
|
"learning_rate": 4.791048600417824e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2244,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.21872518588031473,
|
|
"grad_norm": 8.125306615010935,
|
|
"learning_rate": 4.788520021313416e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.21944705118024976,
|
|
"grad_norm": 9.119966563903791,
|
|
"learning_rate": 4.785976910044255e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2032,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.2201689164801848,
|
|
"grad_norm": 12.1919687251649,
|
|
"learning_rate": 4.783419282759168e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2312,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.22089078178011984,
|
|
"grad_norm": 10.966337886877598,
|
|
"learning_rate": 4.780847155699157e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2282,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.22161264708005488,
|
|
"grad_norm": 10.89546424385192,
|
|
"learning_rate": 4.778260545197301e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2326,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.22233451237998988,
|
|
"grad_norm": 8.635404642662332,
|
|
"learning_rate": 4.775659467678645e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.22305637767992492,
|
|
"grad_norm": 13.172555524642066,
|
|
"learning_rate": 4.773043939660105e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2527,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.22377824297985996,
|
|
"grad_norm": 10.053263971609315,
|
|
"learning_rate": 4.770413977750353e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.224500108279795,
|
|
"grad_norm": 7.080435865876819,
|
|
"learning_rate": 4.7677695986497225e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2285,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.22522197357973003,
|
|
"grad_norm": 8.489071949818994,
|
|
"learning_rate": 4.7651108191500896e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2217,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.22594383887966504,
|
|
"grad_norm": 7.48950614324499,
|
|
"learning_rate": 4.762437656134778e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.22666570417960008,
|
|
"grad_norm": 11.147541103318718,
|
|
"learning_rate": 4.7597501265784466e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.22738756947953512,
|
|
"grad_norm": 9.2136696723812,
|
|
"learning_rate": 4.757048247546982e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2588,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.22810943477947015,
|
|
"grad_norm": 12.83236893491969,
|
|
"learning_rate": 4.7543320361973884e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2172,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.2288313000794052,
|
|
"grad_norm": 9.608775546642143,
|
|
"learning_rate": 4.751601509777683e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.2295531653793402,
|
|
"grad_norm": 8.893320591024583,
|
|
"learning_rate": 4.748856685626784e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2299,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.23027503067927524,
|
|
"grad_norm": 6.813416083172801,
|
|
"learning_rate": 4.7460975811743986e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2321,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.23099689597921028,
|
|
"grad_norm": 11.910753128704322,
|
|
"learning_rate": 4.743324213940917e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.2317187612791453,
|
|
"grad_norm": 10.880683342821822,
|
|
"learning_rate": 4.740536601537295e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.226,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.23244062657908035,
|
|
"grad_norm": 9.387075119803855,
|
|
"learning_rate": 4.73773476166495e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2052,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.2331624918790154,
|
|
"grad_norm": 9.319384117903645,
|
|
"learning_rate": 4.73491871211564e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2334,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.2338843571789504,
|
|
"grad_norm": 7.92062494143796,
|
|
"learning_rate": 4.7320884707713573e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.226,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.23460622247888543,
|
|
"grad_norm": 7.5905378566734445,
|
|
"learning_rate": 4.7292440556042116e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2381,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.23532808777882047,
|
|
"grad_norm": 7.104939281324458,
|
|
"learning_rate": 4.726385484676318e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.2360499530787555,
|
|
"grad_norm": 7.112864388089024,
|
|
"learning_rate": 4.723512776139679e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.218,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.23677181837869055,
|
|
"grad_norm": 9.110259152947808,
|
|
"learning_rate": 4.7206259482360734e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2342,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.23749368367862556,
|
|
"grad_norm": 8.883129926158574,
|
|
"learning_rate": 4.7177250192969364e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2037,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.2382155489785606,
|
|
"grad_norm": 10.573842474018138,
|
|
"learning_rate": 4.714810007743247e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2226,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.23893741427849563,
|
|
"grad_norm": 9.209497924010089,
|
|
"learning_rate": 4.7118809320854077e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2122,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.23965927957843067,
|
|
"grad_norm": 9.372117356581454,
|
|
"learning_rate": 4.7089378109231294e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.2403811448783657,
|
|
"grad_norm": 8.352914263375952,
|
|
"learning_rate": 4.7059806629453116e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2278,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.24110301017830074,
|
|
"grad_norm": 10.512336386108508,
|
|
"learning_rate": 4.7030095069299257e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.24182487547823575,
|
|
"grad_norm": 9.166170654186674,
|
|
"learning_rate": 4.700024361743893e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.2425467407781708,
|
|
"grad_norm": 10.052841673647455,
|
|
"learning_rate": 4.697025246342967e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2115,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.24326860607810583,
|
|
"grad_norm": 12.87931292960993,
|
|
"learning_rate": 4.6940121797716127e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.227,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.24399047137804086,
|
|
"grad_norm": 11.246155735848257,
|
|
"learning_rate": 4.6909851811628853e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.2447123366779759,
|
|
"grad_norm": 8.247097831323758,
|
|
"learning_rate": 4.68794426973831e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2203,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.375,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.2454342019779109,
|
|
"grad_norm": 7.982031142460416,
|
|
"learning_rate": 4.684889464807755e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2141,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.24615606727784595,
|
|
"grad_norm": 8.859057538137234,
|
|
"learning_rate": 4.681820785769317e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2002,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.24687793257778098,
|
|
"grad_norm": 10.507595703830146,
|
|
"learning_rate": 4.678738252109192e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2272,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.24759979787771602,
|
|
"grad_norm": 9.701082860121796,
|
|
"learning_rate": 4.675641883401553e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.24832166317765106,
|
|
"grad_norm": 9.249938002509944,
|
|
"learning_rate": 4.672531699308425e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.2490435284775861,
|
|
"grad_norm": 10.341717167410792,
|
|
"learning_rate": 4.669407719579562e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2035,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.2497653937775211,
|
|
"grad_norm": 8.681196124713722,
|
|
"learning_rate": 4.666269964052322e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2126,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.25048725907745617,
|
|
"grad_norm": 9.212928240257371,
|
|
"learning_rate": 4.6631184526515384e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2196,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.2512091243773912,
|
|
"grad_norm": 9.229554206053074,
|
|
"learning_rate": 4.6599532053893936e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2172,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.2519309896773262,
|
|
"grad_norm": 7.2543396832627876,
|
|
"learning_rate": 4.6567742423652955e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2158,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.25265285497726125,
|
|
"grad_norm": 8.156746365270518,
|
|
"learning_rate": 4.6535815837657456e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.25337472027719626,
|
|
"grad_norm": 9.476042547482471,
|
|
"learning_rate": 4.6503752498642133e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2247,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.25409658557713133,
|
|
"grad_norm": 11.068802868497535,
|
|
"learning_rate": 4.6471552610210073e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2236,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.25481845087706634,
|
|
"grad_norm": 9.184379291366328,
|
|
"learning_rate": 4.6439216376831447e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2076,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.25554031617700135,
|
|
"grad_norm": 12.322007532110128,
|
|
"learning_rate": 4.6406744003842213e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2124,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.2562621814769364,
|
|
"grad_norm": 7.159317606243659,
|
|
"learning_rate": 4.6374135697442833e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2098,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.2569840467768714,
|
|
"grad_norm": 29.4711769624419,
|
|
"learning_rate": 4.634139166469695e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2267,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.2577059120768065,
|
|
"grad_norm": 7.80532559868148,
|
|
"learning_rate": 4.6308512113530063e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2187,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.0,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.2584277773767415,
|
|
"grad_norm": 7.185706232473536,
|
|
"learning_rate": 4.627549725272822e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.2591496426766765,
|
|
"grad_norm": 7.836489814314065,
|
|
"learning_rate": 4.62423472919367e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2143,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.25987150797661157,
|
|
"grad_norm": 10.312658330041376,
|
|
"learning_rate": 4.6209062441658654e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2176,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.2605933732765466,
|
|
"grad_norm": 7.660185299883346,
|
|
"learning_rate": 4.6175642913253783e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.26131523857648165,
|
|
"grad_norm": 9.630986662907384,
|
|
"learning_rate": 4.6142088918937016e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.26203710387641665,
|
|
"grad_norm": 7.9637824352647195,
|
|
"learning_rate": 4.6108400671777135e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.2627589691763517,
|
|
"grad_norm": 10.302801071286787,
|
|
"learning_rate": 4.607457838569544e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.208,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.26348083447628673,
|
|
"grad_norm": 9.200762753858287,
|
|
"learning_rate": 4.6040622275464355e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.26420269977622174,
|
|
"grad_norm": 8.72614280680941,
|
|
"learning_rate": 4.6006532556706124e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2263,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.2649245650761568,
|
|
"grad_norm": 9.95080301383368,
|
|
"learning_rate": 4.5972309445891386e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2212,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.2656464303760918,
|
|
"grad_norm": 7.898181548134635,
|
|
"learning_rate": 4.593795316033783e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2504,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.2663682956760269,
|
|
"grad_norm": 9.27071552978421,
|
|
"learning_rate": 4.5903463918208815e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2182,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.2670901609759619,
|
|
"grad_norm": 7.471297940396982,
|
|
"learning_rate": 4.586884193851197e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.2678120262758969,
|
|
"grad_norm": 9.198139304204727,
|
|
"learning_rate": 4.5834087441097806e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1971,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.26853389157583196,
|
|
"grad_norm": 7.074538199167009,
|
|
"learning_rate": 4.5799200646658345e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.26925575687576697,
|
|
"grad_norm": 7.9353245918675,
|
|
"learning_rate": 4.576418177672568e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2194,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.26997762217570204,
|
|
"grad_norm": 7.362232757163594,
|
|
"learning_rate": 4.5729031053670596e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2025,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.27069948747563705,
|
|
"grad_norm": 9.803377167019393,
|
|
"learning_rate": 4.569374870070114e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.27142135277557206,
|
|
"grad_norm": 6.923819353668018,
|
|
"learning_rate": 4.565833494186122e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1945,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.2721432180755071,
|
|
"grad_norm": 7.409462373704994,
|
|
"learning_rate": 4.562279000202919e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.27286508337544213,
|
|
"grad_norm": 8.55425171800717,
|
|
"learning_rate": 4.5587114106916366e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.2735869486753772,
|
|
"grad_norm": 8.283760196639793,
|
|
"learning_rate": 4.5551307483065664e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.2743088139753122,
|
|
"grad_norm": 9.334259623082545,
|
|
"learning_rate": 4.5515370357850136e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.2750306792752472,
|
|
"grad_norm": 8.790752805238721,
|
|
"learning_rate": 4.547930295947151e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.2757525445751823,
|
|
"grad_norm": 11.07615206391536,
|
|
"learning_rate": 4.5443105516958737e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.2764744098751173,
|
|
"grad_norm": 12.156443108793454,
|
|
"learning_rate": 4.5406778260166586e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2064,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.27719627517505235,
|
|
"grad_norm": 9.054451472292419,
|
|
"learning_rate": 4.5370321419774117e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2138,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.27791814047498736,
|
|
"grad_norm": 6.698493216999741,
|
|
"learning_rate": 4.5333735227283274e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.2786400057749224,
|
|
"grad_norm": 9.735985205726063,
|
|
"learning_rate": 4.5297019915017375e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1875,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.27936187107485744,
|
|
"grad_norm": 8.810687694126111,
|
|
"learning_rate": 4.5260175716119655e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.212,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.28008373637479245,
|
|
"grad_norm": 10.6784883112821,
|
|
"learning_rate": 4.522320286455179e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.2808056016747275,
|
|
"grad_norm": 9.94830452756906,
|
|
"learning_rate": 4.5186101595092403e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.2815274669746625,
|
|
"grad_norm": 9.418299320110647,
|
|
"learning_rate": 4.5148872143335566e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.2822493322745976,
|
|
"grad_norm": 11.904953591713511,
|
|
"learning_rate": 4.5111514745689307e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.2829711975745326,
|
|
"grad_norm": 11.15522188195295,
|
|
"learning_rate": 4.507402963937414e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.2836930628744676,
|
|
"grad_norm": 11.36508327799745,
|
|
"learning_rate": 4.5036417062421506e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2226,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.28441492817440267,
|
|
"grad_norm": 9.897929517208977,
|
|
"learning_rate": 4.4998677253672297e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2208,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.2851367934743377,
|
|
"grad_norm": 9.778813209573705,
|
|
"learning_rate": 4.496081045277533e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.28585865877427274,
|
|
"grad_norm": 7.8313055942532275,
|
|
"learning_rate": 4.492281690018583e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -9.125,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.28658052407420775,
|
|
"grad_norm": 8.317207474938273,
|
|
"learning_rate": 4.4884696837163905e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2117,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.28730238937414276,
|
|
"grad_norm": 10.26976433894447,
|
|
"learning_rate": 4.484645050577299e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2079,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.28802425467407783,
|
|
"grad_norm": 11.840261029137109,
|
|
"learning_rate": 4.480807814887833e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.214,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.28874611997401284,
|
|
"grad_norm": 8.811651662253073,
|
|
"learning_rate": 4.476958001014544e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2135,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.2894679852739479,
|
|
"grad_norm": 8.267204182411973,
|
|
"learning_rate": 4.4730956334038565e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.2901898505738829,
|
|
"grad_norm": 7.77369954894956,
|
|
"learning_rate": 4.46922073658191e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.2909117158738179,
|
|
"grad_norm": 8.886847365802758,
|
|
"learning_rate": 4.465333335154406e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2115,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.291633581173753,
|
|
"grad_norm": 11.540304268095925,
|
|
"learning_rate": 4.461433453806449e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.292355446473688,
|
|
"grad_norm": 11.614291076457404,
|
|
"learning_rate": 4.457521117302392e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.29307731177362306,
|
|
"grad_norm": 11.959955939062517,
|
|
"learning_rate": 4.45359635048568e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.29379917707355807,
|
|
"grad_norm": 10.625522954210599,
|
|
"learning_rate": 4.4496591782786883e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.2562,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.2945210423734931,
|
|
"grad_norm": 8.706370989411305,
|
|
"learning_rate": 4.44570962568257e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2059,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.29524290767342815,
|
|
"grad_norm": 10.084781546011934,
|
|
"learning_rate": 4.4417477177770905e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2059,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.29596477297336315,
|
|
"grad_norm": 10.211730254994997,
|
|
"learning_rate": 4.4377734797204747e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.2966866382732982,
|
|
"grad_norm": 8.821514902632222,
|
|
"learning_rate": 4.433786936749241e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2048,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.29740850357323323,
|
|
"grad_norm": 9.210606252468882,
|
|
"learning_rate": 4.429788114178049e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.2981303688731683,
|
|
"grad_norm": 8.00555161785549,
|
|
"learning_rate": 4.425777037399529e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1886,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.2988522341731033,
|
|
"grad_norm": 11.207346805964308,
|
|
"learning_rate": 4.42175373188413e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.2995740994730383,
|
|
"grad_norm": 8.172701676742552,
|
|
"learning_rate": 4.4177182231799513e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2072,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.3002959647729734,
|
|
"grad_norm": 5.801373290887345,
|
|
"learning_rate": 4.413670536912584e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.3010178300729084,
|
|
"grad_norm": 6.307391224443462,
|
|
"learning_rate": 4.4096106987849457e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.30173969537284345,
|
|
"grad_norm": 11.867957943680011,
|
|
"learning_rate": 4.405538734577121e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.30246156067277846,
|
|
"grad_norm": 9.987667285927838,
|
|
"learning_rate": 4.401454670146193e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.2163,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.30318342597271347,
|
|
"grad_norm": 8.977809465407699,
|
|
"learning_rate": 4.3973585314260836e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1906,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.30390529127264854,
|
|
"grad_norm": 10.825945402642533,
|
|
"learning_rate": 4.393250344427385e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.30462715657258355,
|
|
"grad_norm": 8.126172608631684,
|
|
"learning_rate": 4.389130135237196e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2207,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.3053490218725186,
|
|
"grad_norm": 7.626491220106016,
|
|
"learning_rate": 4.38499793001896e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.3060708871724536,
|
|
"grad_norm": 9.904614258788264,
|
|
"learning_rate": 4.3808537550122913e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.211,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.30679275247238863,
|
|
"grad_norm": 9.163789602486757,
|
|
"learning_rate": 4.376697636532816e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.3075146177723237,
|
|
"grad_norm": 9.658094123529755,
|
|
"learning_rate": 4.3725296009719985e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.3082364830722587,
|
|
"grad_norm": 6.663743741837934,
|
|
"learning_rate": 4.3683496747969804e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1913,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.30895834837219377,
|
|
"grad_norm": 9.431783701004399,
|
|
"learning_rate": 4.364157884550406e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.2088,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.3096802136721288,
|
|
"grad_norm": 10.281531783875531,
|
|
"learning_rate": 4.359954256850259e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.3104020789720638,
|
|
"grad_norm": 9.63227220127926,
|
|
"learning_rate": 4.35573881838969e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.31112394427199885,
|
|
"grad_norm": 10.289092693608229,
|
|
"learning_rate": 4.3515115959368476e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.31184580957193386,
|
|
"grad_norm": 10.974607276803276,
|
|
"learning_rate": 4.3472726163347116e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2179,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.31256767487186893,
|
|
"grad_norm": 6.412992045987879,
|
|
"learning_rate": 4.3430219065009177e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.31328954017180394,
|
|
"grad_norm": 10.403220300532743,
|
|
"learning_rate": 4.3387594934275896e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.31401140547173895,
|
|
"grad_norm": 6.73518127809711,
|
|
"learning_rate": 4.334485404181167e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.314733270771674,
|
|
"grad_norm": 10.054893296825982,
|
|
"learning_rate": 4.3301996659022334e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2176,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.315455136071609,
|
|
"grad_norm": 7.828814625819192,
|
|
"learning_rate": 4.3259023058053444e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2147,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.3161770013715441,
|
|
"grad_norm": 7.009502435156667,
|
|
"learning_rate": 4.3215933511788544e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2081,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.3168988666714791,
|
|
"grad_norm": 9.3284353021479,
|
|
"learning_rate": 4.317272829384743e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.31762073197141416,
|
|
"grad_norm": 8.811880524775491,
|
|
"learning_rate": 4.3129407678584414e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.31834259727134917,
|
|
"grad_norm": 8.125525464636453,
|
|
"learning_rate": 4.3085971941086585e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1915,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.3190644625712842,
|
|
"grad_norm": 8.751734979718838,
|
|
"learning_rate": 4.3042421357172076e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.31978632787121924,
|
|
"grad_norm": 7.608014094617588,
|
|
"learning_rate": 4.2998756203388285e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2152,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.32050819317115425,
|
|
"grad_norm": 7.639743064757877,
|
|
"learning_rate": 4.2954976757010133e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.3212300584710893,
|
|
"grad_norm": 11.460888477745158,
|
|
"learning_rate": 4.2911083296038285e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.32195192377102433,
|
|
"grad_norm": 9.224352773153688,
|
|
"learning_rate": 4.2867076099197446e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1895,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.32267378907095934,
|
|
"grad_norm": 9.244876887068662,
|
|
"learning_rate": 4.2822955445934505e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1772,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.3233956543708944,
|
|
"grad_norm": 8.906087322512382,
|
|
"learning_rate": 4.277872161641681e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.3241175196708294,
|
|
"grad_norm": 9.689414725700667,
|
|
"learning_rate": 4.273437489153041e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.3248393849707645,
|
|
"grad_norm": 8.996613366246144,
|
|
"learning_rate": 4.2689915552878207e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.3255612502706995,
|
|
"grad_norm": 9.607472232534843,
|
|
"learning_rate": 4.2645343882778215e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.3262831155706345,
|
|
"grad_norm": 8.846494209878479,
|
|
"learning_rate": 4.260066016426177e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.32700498087056956,
|
|
"grad_norm": 11.245824423136213,
|
|
"learning_rate": 4.25558646810717e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1856,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.32772684617050457,
|
|
"grad_norm": 8.60581993865266,
|
|
"learning_rate": 4.251095771766055e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.177,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.32844871147043964,
|
|
"grad_norm": 7.6110846227784625,
|
|
"learning_rate": 4.246593955918877e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2188,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.32917057677037465,
|
|
"grad_norm": 7.571006600020749,
|
|
"learning_rate": 4.2420810491522896e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2004,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.32989244207030965,
|
|
"grad_norm": 9.619717180276336,
|
|
"learning_rate": 4.237557080123373e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2037,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.3306143073702447,
|
|
"grad_norm": 9.813397994219041,
|
|
"learning_rate": 4.2330220775594567e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1992,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.33133617267017973,
|
|
"grad_norm": 8.765749359430536,
|
|
"learning_rate": 4.228476070257929e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.3320580379701148,
|
|
"grad_norm": 10.82246666876482,
|
|
"learning_rate": 4.223919087086062e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1802,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.3327799032700498,
|
|
"grad_norm": 7.941514770204173,
|
|
"learning_rate": 4.2193511569808225e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1991,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.3335017685699848,
|
|
"grad_norm": 7.912745662185516,
|
|
"learning_rate": 4.214772308948693e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.3342236338699199,
|
|
"grad_norm": 10.9521651772932,
|
|
"learning_rate": 4.2101825720654827e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2081,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.3349454991698549,
|
|
"grad_norm": 6.99023599526195,
|
|
"learning_rate": 4.205581975476146e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2041,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.33566736446978995,
|
|
"grad_norm": 8.603344530140768,
|
|
"learning_rate": 4.200970548394598e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1965,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.33638922976972496,
|
|
"grad_norm": 9.717823679934542,
|
|
"learning_rate": 4.196348320103527e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.33711109506966,
|
|
"grad_norm": 7.935814772911749,
|
|
"learning_rate": 4.191715319954209e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.33783296036959504,
|
|
"grad_norm": 8.769171773282734,
|
|
"learning_rate": 4.187071577366321e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.33855482566953005,
|
|
"grad_norm": 7.096843320580391,
|
|
"learning_rate": 4.182417121827755e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1906,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.3392766909694651,
|
|
"grad_norm": 10.058671513695627,
|
|
"learning_rate": 4.177751982894433e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.1903,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.3399985562694001,
|
|
"grad_norm": 10.565296750226013,
|
|
"learning_rate": 4.1730761901901135e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1993,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.3407204215693352,
|
|
"grad_norm": 9.850412180420557,
|
|
"learning_rate": 4.168389773406209e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.3414422868692702,
|
|
"grad_norm": 11.906676776121467,
|
|
"learning_rate": 4.1636927623015927e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.3421641521692052,
|
|
"grad_norm": 10.205674489245993,
|
|
"learning_rate": 4.158985186702415e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2092,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.34288601746914027,
|
|
"grad_norm": 9.260984330253814,
|
|
"learning_rate": 4.1542670765019104e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.3436078827690753,
|
|
"grad_norm": 8.949027397431555,
|
|
"learning_rate": 4.149538461660206e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.34432974806901034,
|
|
"grad_norm": 6.720014771416349,
|
|
"learning_rate": 4.144799372204136e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1826,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.34505161336894535,
|
|
"grad_norm": 8.959024936705582,
|
|
"learning_rate": 4.140049838227049e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1987,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.34577347866888036,
|
|
"grad_norm": 10.372748362230645,
|
|
"learning_rate": 4.135289889888615e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.34649534396881543,
|
|
"grad_norm": 8.270570715124181,
|
|
"learning_rate": 4.130519557414636e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.34721720926875044,
|
|
"grad_norm": 9.893879868420212,
|
|
"learning_rate": 4.1257388710968533e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1925,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.3479390745686855,
|
|
"grad_norm": 12.331591426723168,
|
|
"learning_rate": 4.120947861292757e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.3486609398686205,
|
|
"grad_norm": 10.172376095851071,
|
|
"learning_rate": 4.11614655842539e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.3493828051685555,
|
|
"grad_norm": 11.310294528639917,
|
|
"learning_rate": 4.111334992983156e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1973,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.3501046704684906,
|
|
"grad_norm": 9.138376308040302,
|
|
"learning_rate": 4.1065131955196294e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.3508265357684256,
|
|
"grad_norm": 10.948111955434124,
|
|
"learning_rate": 4.1016811966533536e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1823,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.35154840106836066,
|
|
"grad_norm": 6.8483776654650965,
|
|
"learning_rate": 4.096839027067656e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2069,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.35227026636829567,
|
|
"grad_norm": 7.262308675444548,
|
|
"learning_rate": 4.0919867175104435e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.35299213166823074,
|
|
"grad_norm": 17.175486759838638,
|
|
"learning_rate": 4.0871242987940155e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1981,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.35371399696816574,
|
|
"grad_norm": 6.418913714694265,
|
|
"learning_rate": 4.082251801794865e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.35443586226810075,
|
|
"grad_norm": 6.8125539564399515,
|
|
"learning_rate": 4.0773692574534795e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1871,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.3551577275680358,
|
|
"grad_norm": 7.814190613362822,
|
|
"learning_rate": 4.072476696774151e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1852,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.35587959286797083,
|
|
"grad_norm": 10.826610307359978,
|
|
"learning_rate": 4.0675741508247715e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.3566014581679059,
|
|
"grad_norm": 8.596687269801581,
|
|
"learning_rate": 4.062661650736643e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1869,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.3573233234678409,
|
|
"grad_norm": 6.416195679478135,
|
|
"learning_rate": 4.0577392277042766e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1855,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.3580451887677759,
|
|
"grad_norm": 8.918928643083534,
|
|
"learning_rate": 4.0528069129851914e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1846,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.358767054067711,
|
|
"grad_norm": 8.47545192360217,
|
|
"learning_rate": 4.0478647378997223e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2025,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.359488919367646,
|
|
"grad_norm": 8.09646150016637,
|
|
"learning_rate": 4.0429127338308154e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.206,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.36021078466758105,
|
|
"grad_norm": 7.912242395511905,
|
|
"learning_rate": 4.037950932223832e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.36093264996751606,
|
|
"grad_norm": 8.510565195808283,
|
|
"learning_rate": 4.032979364586349e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2007,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.36093264996751606,
|
|
"eval_logits/chosen": -2.90625,
|
|
"eval_logits/rejected": -2.65625,
|
|
"eval_logps/chosen": -700.0,
|
|
"eval_logps/rejected": -1040.0,
|
|
"eval_loss": 0.22352416813373566,
|
|
"eval_rewards/accuracies": 0.9081980586051941,
|
|
"eval_rewards/chosen": -5.3125,
|
|
"eval_rewards/margins": 3.546875,
|
|
"eval_rewards/rejected": -8.875,
|
|
"eval_runtime": 3594.5422,
|
|
"eval_samples_per_second": 27.403,
|
|
"eval_steps_per_second": 0.428,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.36165451526745107,
|
|
"grad_norm": 9.508216962196549,
|
|
"learning_rate": 4.027998062487955e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"epoch": 0.36237638056738614,
|
|
"grad_norm": 11.433083041803624,
|
|
"learning_rate": 4.023007057560057e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"epoch": 0.36309824586732115,
|
|
"grad_norm": 9.561775751013597,
|
|
"learning_rate": 4.018006381495671e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1764,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"epoch": 0.3638201111672562,
|
|
"grad_norm": 8.343880295640096,
|
|
"learning_rate": 4.012996066049229e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1988,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"epoch": 0.3645419764671912,
|
|
"grad_norm": 7.788342844437915,
|
|
"learning_rate": 4.0079761430363676e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1751,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"epoch": 0.36526384176712623,
|
|
"grad_norm": 9.014324627097576,
|
|
"learning_rate": 4.002946644333739e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"epoch": 0.3659857070670613,
|
|
"grad_norm": 10.445201108593542,
|
|
"learning_rate": 3.997907601878796e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1911,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"epoch": 0.3667075723669963,
|
|
"grad_norm": 8.250001844997403,
|
|
"learning_rate": 3.992859047669596e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"epoch": 0.36742943766693137,
|
|
"grad_norm": 11.753669122881204,
|
|
"learning_rate": 3.987801013764596e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1879,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -6.28125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"epoch": 0.3681513029668664,
|
|
"grad_norm": 12.658837897492882,
|
|
"learning_rate": 3.9827335322824496e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"epoch": 0.3688731682668014,
|
|
"grad_norm": 7.844631820767091,
|
|
"learning_rate": 3.977656635401805e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"epoch": 0.36959503356673645,
|
|
"grad_norm": 7.842154920577124,
|
|
"learning_rate": 3.972570355361093e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"epoch": 0.37031689886667146,
|
|
"grad_norm": 12.429197801825422,
|
|
"learning_rate": 3.967474724458334e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"epoch": 0.3710387641666065,
|
|
"grad_norm": 9.022738866247183,
|
|
"learning_rate": 3.962369775050922e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"epoch": 0.37176062946654154,
|
|
"grad_norm": 7.735894816170176,
|
|
"learning_rate": 3.957255539555426e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"epoch": 0.3724824947664766,
|
|
"grad_norm": 7.904648881093523,
|
|
"learning_rate": 3.9521320504473804e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1898,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"epoch": 0.3732043600664116,
|
|
"grad_norm": 8.068500078950347,
|
|
"learning_rate": 3.9469993402610823e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"epoch": 0.3739262253663466,
|
|
"grad_norm": 9.258484145773922,
|
|
"learning_rate": 3.941857441589382e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"epoch": 0.3746480906662817,
|
|
"grad_norm": 9.720892187785136,
|
|
"learning_rate": 3.936706387083476e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"epoch": 0.3753699559662167,
|
|
"grad_norm": 5.5288559635381365,
|
|
"learning_rate": 3.9315462094527006e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.37609182126615176,
|
|
"grad_norm": 7.613626077988378,
|
|
"learning_rate": 3.9263769414643273e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1846,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"epoch": 0.37681368656608677,
|
|
"grad_norm": 6.807486326402407,
|
|
"learning_rate": 3.9211986159433477e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"epoch": 0.3775355518660218,
|
|
"grad_norm": 9.160049813590764,
|
|
"learning_rate": 3.91601126577227e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1827,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"epoch": 0.37825741716595684,
|
|
"grad_norm": 10.144956306040507,
|
|
"learning_rate": 3.910814923890911e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1734,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"epoch": 0.37897928246589185,
|
|
"grad_norm": 10.789604509188436,
|
|
"learning_rate": 3.9056096232961834e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1953,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"epoch": 0.3797011477658269,
|
|
"grad_norm": 8.03159337115462,
|
|
"learning_rate": 3.9003953970418877e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1836,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"epoch": 0.38042301306576193,
|
|
"grad_norm": 9.599822536896923,
|
|
"learning_rate": 3.8951722782385046e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1911,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"epoch": 0.38114487836569694,
|
|
"grad_norm": 10.071856627849183,
|
|
"learning_rate": 3.8899403000529803e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"epoch": 0.381866743665632,
|
|
"grad_norm": 10.524225607275767,
|
|
"learning_rate": 3.8846994957085194e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"epoch": 0.382588608965567,
|
|
"grad_norm": 8.616712400688618,
|
|
"learning_rate": 3.879449898484374e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1873,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"epoch": 0.3833104742655021,
|
|
"grad_norm": 8.45520493106579,
|
|
"learning_rate": 3.874191541715629e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1777,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"epoch": 0.3840323395654371,
|
|
"grad_norm": 6.786490880185805,
|
|
"learning_rate": 3.868924458792994e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"epoch": 0.3847542048653721,
|
|
"grad_norm": 9.306230356711307,
|
|
"learning_rate": 3.86364868316259e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1961,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"epoch": 0.38547607016530716,
|
|
"grad_norm": 6.874539829983695,
|
|
"learning_rate": 3.8583642483257374e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"epoch": 0.38619793546524217,
|
|
"grad_norm": 9.535981769729352,
|
|
"learning_rate": 3.85307118783874e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"epoch": 0.38691980076517724,
|
|
"grad_norm": 7.916792530196583,
|
|
"learning_rate": 3.8477695353126785e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1841,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"epoch": 0.38764166606511224,
|
|
"grad_norm": 11.021130141780702,
|
|
"learning_rate": 3.8424593244131897e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1947,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"epoch": 0.3883635313650473,
|
|
"grad_norm": 8.203256258991978,
|
|
"learning_rate": 3.83714058886026e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"epoch": 0.3890853966649823,
|
|
"grad_norm": 7.923518332280399,
|
|
"learning_rate": 3.8318133624280046e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"epoch": 0.38980726196491733,
|
|
"grad_norm": 13.720539493582635,
|
|
"learning_rate": 3.826477678944457e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1945,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.3905291272648524,
|
|
"grad_norm": 7.1393388118343495,
|
|
"learning_rate": 3.821133572291354e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1814,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"epoch": 0.3912509925647874,
|
|
"grad_norm": 9.329246298724282,
|
|
"learning_rate": 3.8157810764039187e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1938,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"epoch": 0.39197285786472247,
|
|
"grad_norm": 6.811570201519407,
|
|
"learning_rate": 3.810420225270647e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"epoch": 0.3926947231646575,
|
|
"grad_norm": 6.663922484272897,
|
|
"learning_rate": 3.80505105293309e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1847,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"epoch": 0.3934165884645925,
|
|
"grad_norm": 9.592577634918783,
|
|
"learning_rate": 3.799673593485638e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1773,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"epoch": 0.39413845376452755,
|
|
"grad_norm": 9.134235793523999,
|
|
"learning_rate": 3.794287881075307e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1741,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"epoch": 0.39486031906446256,
|
|
"grad_norm": 8.925533209727694,
|
|
"learning_rate": 3.788893949901517e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"epoch": 0.3955821843643976,
|
|
"grad_norm": 7.220703628171884,
|
|
"learning_rate": 3.783491834215879e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"epoch": 0.39630404966433264,
|
|
"grad_norm": 10.56460893328502,
|
|
"learning_rate": 3.778081568321976e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"epoch": 0.39702591496426765,
|
|
"grad_norm": 10.362564107960075,
|
|
"learning_rate": 3.772663186575143e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 0.3977477802642027,
|
|
"grad_norm": 9.293115038985665,
|
|
"learning_rate": 3.7672367233822537e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"epoch": 0.3984696455641377,
|
|
"grad_norm": 8.820973054087007,
|
|
"learning_rate": 3.761802213201498e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"epoch": 0.3991915108640728,
|
|
"grad_norm": 8.927273915289893,
|
|
"learning_rate": 3.756359690542163e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"epoch": 0.3999133761640078,
|
|
"grad_norm": 9.274501424450483,
|
|
"learning_rate": 3.750909189964417e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"epoch": 0.4006352414639428,
|
|
"grad_norm": 14.27324930632179,
|
|
"learning_rate": 3.7454507460790893e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"epoch": 0.40135710676387787,
|
|
"grad_norm": 5.934956865301077,
|
|
"learning_rate": 3.739984393547446e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1771,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"epoch": 0.4020789720638129,
|
|
"grad_norm": 8.744047306639088,
|
|
"learning_rate": 3.7345101670809755e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"epoch": 0.40280083736374794,
|
|
"grad_norm": 10.141402293967214,
|
|
"learning_rate": 3.7290281014411655e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1761,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"epoch": 0.40352270266368295,
|
|
"grad_norm": 8.951801438600679,
|
|
"learning_rate": 3.723538231439284e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"epoch": 0.40424456796361796,
|
|
"grad_norm": 9.18772270207208,
|
|
"learning_rate": 3.7180405919361547e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1996,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.404966433263553,
|
|
"grad_norm": 11.802072092894162,
|
|
"learning_rate": 3.7125352178419396e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"epoch": 0.40568829856348804,
|
|
"grad_norm": 7.441092805572635,
|
|
"learning_rate": 3.707022144115914e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1979,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"epoch": 0.4064101638634231,
|
|
"grad_norm": 8.542112860478404,
|
|
"learning_rate": 3.701501405766248e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1833,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"epoch": 0.4071320291633581,
|
|
"grad_norm": 8.690977859423688,
|
|
"learning_rate": 3.69597303784978e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"epoch": 0.4078538944632932,
|
|
"grad_norm": 7.133375992414793,
|
|
"learning_rate": 3.690437075471797e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1834,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.0,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"epoch": 0.4085757597632282,
|
|
"grad_norm": 9.060343279065481,
|
|
"learning_rate": 3.684893553785815e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"epoch": 0.4092976250631632,
|
|
"grad_norm": 9.150169701262556,
|
|
"learning_rate": 3.6793425079933446e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"epoch": 0.41001949036309826,
|
|
"grad_norm": 11.459154584979037,
|
|
"learning_rate": 3.67378397334368e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.191,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"epoch": 0.41074135566303327,
|
|
"grad_norm": 10.628103177908564,
|
|
"learning_rate": 3.668217985133668e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1765,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"epoch": 0.41146322096296833,
|
|
"grad_norm": 10.663063859010725,
|
|
"learning_rate": 3.662644578707486e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"epoch": 0.41218508626290334,
|
|
"grad_norm": 9.68068326501746,
|
|
"learning_rate": 3.657063789456418e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1893,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"epoch": 0.41290695156283835,
|
|
"grad_norm": 9.038765481874343,
|
|
"learning_rate": 3.6514756528186283e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1737,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"epoch": 0.4136288168627734,
|
|
"grad_norm": 7.8137408122782315,
|
|
"learning_rate": 3.645880204278936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"epoch": 0.41435068216270843,
|
|
"grad_norm": 10.519994806474793,
|
|
"learning_rate": 3.640277479368594e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"epoch": 0.4150725474626435,
|
|
"grad_norm": 6.618841094002535,
|
|
"learning_rate": 3.6346675136650595e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1686,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"epoch": 0.4157944127625785,
|
|
"grad_norm": 9.449161368315643,
|
|
"learning_rate": 3.629050342791766e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1883,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"epoch": 0.4165162780625135,
|
|
"grad_norm": 8.645062230868888,
|
|
"learning_rate": 3.6234260024179033e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"epoch": 0.4172381433624486,
|
|
"grad_norm": 8.61829308724476,
|
|
"learning_rate": 3.6177945282581866e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"epoch": 0.4179600086623836,
|
|
"grad_norm": 8.790765287429858,
|
|
"learning_rate": 3.6121559560726313e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"epoch": 0.41868187396231865,
|
|
"grad_norm": 10.485975334565037,
|
|
"learning_rate": 3.6065103216663237e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.187,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.41940373926225366,
|
|
"grad_norm": 7.596154187472179,
|
|
"learning_rate": 3.600857660889199e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"epoch": 0.42012560456218867,
|
|
"grad_norm": 7.510040671882969,
|
|
"learning_rate": 3.5951980096358055e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"epoch": 0.42084746986212374,
|
|
"grad_norm": 6.151781029516534,
|
|
"learning_rate": 3.589531403845085e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1812,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"epoch": 0.42156933516205874,
|
|
"grad_norm": 10.349825384153817,
|
|
"learning_rate": 3.5838578795001393e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"epoch": 0.4222912004619938,
|
|
"grad_norm": 8.687369080835087,
|
|
"learning_rate": 3.578177472628002e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.1875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"epoch": 0.4230130657619288,
|
|
"grad_norm": 9.245880790108894,
|
|
"learning_rate": 3.572490219299414e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1933,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"epoch": 0.42373493106186383,
|
|
"grad_norm": 10.513199703384252,
|
|
"learning_rate": 3.5667961556285876e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.172,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"epoch": 0.4244567963617989,
|
|
"grad_norm": 8.067276310939405,
|
|
"learning_rate": 3.561095317772984e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"epoch": 0.4251786616617339,
|
|
"grad_norm": 9.335763045264015,
|
|
"learning_rate": 3.5553877419330797e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"epoch": 0.42590052696166897,
|
|
"grad_norm": 8.536819036424028,
|
|
"learning_rate": 3.5496734643521364e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.198,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"epoch": 0.426622392261604,
|
|
"grad_norm": 10.712349432459751,
|
|
"learning_rate": 3.543952521315975e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1793,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"epoch": 0.42734425756153904,
|
|
"grad_norm": 9.762289940914592,
|
|
"learning_rate": 3.538224949152738e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.179,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"epoch": 0.42806612286147405,
|
|
"grad_norm": 8.279157154276295,
|
|
"learning_rate": 3.5324907842326676e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"epoch": 0.42878798816140906,
|
|
"grad_norm": 7.703443487862446,
|
|
"learning_rate": 3.526750062967866e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"epoch": 0.4295098534613441,
|
|
"grad_norm": 8.915401918749712,
|
|
"learning_rate": 3.5210028218120714e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"epoch": 0.43023171876127914,
|
|
"grad_norm": 7.356754892003317,
|
|
"learning_rate": 3.515249097260422e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1744,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"epoch": 0.4309535840612142,
|
|
"grad_norm": 8.27577072471933,
|
|
"learning_rate": 3.509488925849227e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1813,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"epoch": 0.4316754493611492,
|
|
"grad_norm": 8.91909433201475,
|
|
"learning_rate": 3.503722344155731e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"epoch": 0.4323973146610842,
|
|
"grad_norm": 10.528827951999629,
|
|
"learning_rate": 3.4979493887978864e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"epoch": 0.4331191799610193,
|
|
"grad_norm": 6.577916440055691,
|
|
"learning_rate": 3.4921700964341173e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.4338410452609543,
|
|
"grad_norm": 9.810703167678307,
|
|
"learning_rate": 3.486384503763089e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"epoch": 0.43456291056088936,
|
|
"grad_norm": 7.569455182394066,
|
|
"learning_rate": 3.480592647523472e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"epoch": 0.43528477586082437,
|
|
"grad_norm": 6.998884892266847,
|
|
"learning_rate": 3.4747945644937133e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"epoch": 0.4360066411607594,
|
|
"grad_norm": 7.151254090045432,
|
|
"learning_rate": 3.468990291491799e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1727,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"epoch": 0.43672850646069444,
|
|
"grad_norm": 6.561607683446632,
|
|
"learning_rate": 3.46317986537502e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1876,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"epoch": 0.43745037176062945,
|
|
"grad_norm": 6.7139191061366805,
|
|
"learning_rate": 3.4573633230397414e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"epoch": 0.4381722370605645,
|
|
"grad_norm": 8.411313986797488,
|
|
"learning_rate": 3.451540701421167e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"epoch": 0.4388941023604995,
|
|
"grad_norm": 8.23452700502391,
|
|
"learning_rate": 3.445712037493104e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"epoch": 0.43961596766043454,
|
|
"grad_norm": 7.143620144559741,
|
|
"learning_rate": 3.439877368267725e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1989,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"epoch": 0.4403378329603696,
|
|
"grad_norm": 5.802482964711945,
|
|
"learning_rate": 3.434036730795342e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"epoch": 0.4410596982603046,
|
|
"grad_norm": 8.719522994114646,
|
|
"learning_rate": 3.428190162164162e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1428,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"epoch": 0.4417815635602397,
|
|
"grad_norm": 8.401165390196324,
|
|
"learning_rate": 3.422337699500056e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"epoch": 0.4425034288601747,
|
|
"grad_norm": 7.273765835778647,
|
|
"learning_rate": 3.4164793799663207e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"epoch": 0.44322529416010975,
|
|
"grad_norm": 11.508967977205781,
|
|
"learning_rate": 3.410615240763446e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2057,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"epoch": 0.44394715946004476,
|
|
"grad_norm": 9.492547090583608,
|
|
"learning_rate": 3.4047453191288775e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"epoch": 0.44466902475997977,
|
|
"grad_norm": 6.826894009448305,
|
|
"learning_rate": 3.3988696523367777e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"epoch": 0.44539089005991483,
|
|
"grad_norm": 7.908259059531054,
|
|
"learning_rate": 3.3929882776977905e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1855,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"epoch": 0.44611275535984984,
|
|
"grad_norm": 9.323313899318885,
|
|
"learning_rate": 3.387101232558807e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"epoch": 0.4468346206597849,
|
|
"grad_norm": 10.574159066231,
|
|
"learning_rate": 3.381208554302723e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"epoch": 0.4475564859597199,
|
|
"grad_norm": 9.45659923763495,
|
|
"learning_rate": 3.3753102803482094e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.44827835125965493,
|
|
"grad_norm": 7.8086390499161045,
|
|
"learning_rate": 3.3694064481494655e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"epoch": 0.44900021655959,
|
|
"grad_norm": 6.5084303039612985,
|
|
"learning_rate": 3.3634970951959867e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1641,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"epoch": 0.449722081859525,
|
|
"grad_norm": 8.137911391940417,
|
|
"learning_rate": 3.357582259012327e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1842,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"epoch": 0.45044394715946007,
|
|
"grad_norm": 8.376896629675258,
|
|
"learning_rate": 3.351661977157859e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"epoch": 0.4511658124593951,
|
|
"grad_norm": 10.56542182965613,
|
|
"learning_rate": 3.3457362872265324e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1604,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"epoch": 0.4518876777593301,
|
|
"grad_norm": 7.833165560372883,
|
|
"learning_rate": 3.339805226846642e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"epoch": 0.45260954305926515,
|
|
"grad_norm": 7.8349709497727025,
|
|
"learning_rate": 3.3338688336805823e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"epoch": 0.45333140835920016,
|
|
"grad_norm": 9.092744073048326,
|
|
"learning_rate": 3.327927145424613e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"epoch": 0.4540532736591352,
|
|
"grad_norm": 10.949550588632437,
|
|
"learning_rate": 3.321980199808616e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.2021,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"epoch": 0.45477513895907024,
|
|
"grad_norm": 10.844258468917948,
|
|
"learning_rate": 3.316028034595861e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"epoch": 0.45549700425900524,
|
|
"grad_norm": 13.438072021230731,
|
|
"learning_rate": 3.3100706875827576e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1739,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"epoch": 0.4562188695589403,
|
|
"grad_norm": 10.43903315431466,
|
|
"learning_rate": 3.304108196598624e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"epoch": 0.4569407348588753,
|
|
"grad_norm": 9.193341851899063,
|
|
"learning_rate": 3.29814059950544e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"epoch": 0.4576626001588104,
|
|
"grad_norm": 9.553793370343385,
|
|
"learning_rate": 3.2921679341976104e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"epoch": 0.4583844654587454,
|
|
"grad_norm": 9.073658565124525,
|
|
"learning_rate": 3.286190238601725e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"epoch": 0.4591063307586804,
|
|
"grad_norm": 10.747256513722208,
|
|
"learning_rate": 3.280207550676312e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1773,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"epoch": 0.45982819605861547,
|
|
"grad_norm": 10.072254030204776,
|
|
"learning_rate": 3.274219908411605e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1759,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"epoch": 0.4605500613585505,
|
|
"grad_norm": 9.55913362196861,
|
|
"learning_rate": 3.2682273498292957e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"epoch": 0.46127192665848554,
|
|
"grad_norm": 7.6694528016919055,
|
|
"learning_rate": 3.262229912982295e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"epoch": 0.46199379195842055,
|
|
"grad_norm": 10.685410106690007,
|
|
"learning_rate": 3.2562276359544917e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.4627156572583556,
|
|
"grad_norm": 6.117106649506752,
|
|
"learning_rate": 3.2502205568605087e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.167,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"epoch": 0.4634375225582906,
|
|
"grad_norm": 10.7792422427223,
|
|
"learning_rate": 3.244208713845461e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1907,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"epoch": 0.46415938785822564,
|
|
"grad_norm": 7.813563275474187,
|
|
"learning_rate": 3.2381921450847187e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"epoch": 0.4648812531581607,
|
|
"grad_norm": 10.042020947007074,
|
|
"learning_rate": 3.2321708887836573e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"epoch": 0.4656031184580957,
|
|
"grad_norm": 10.113981129252544,
|
|
"learning_rate": 3.226144983177419e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1802,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"epoch": 0.4663249837580308,
|
|
"grad_norm": 10.153352236033115,
|
|
"learning_rate": 3.220114466530668e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1811,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"epoch": 0.4670468490579658,
|
|
"grad_norm": 10.905089224037248,
|
|
"learning_rate": 3.214079377137352e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1801,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"epoch": 0.4677687143579008,
|
|
"grad_norm": 8.04177832973184,
|
|
"learning_rate": 3.2080397533204526e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"epoch": 0.46849057965783586,
|
|
"grad_norm": 13.034607545233241,
|
|
"learning_rate": 3.201995633431747e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"epoch": 0.46921244495777087,
|
|
"grad_norm": 6.884563442702245,
|
|
"learning_rate": 3.1959470558515613e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 0.46993431025770593,
|
|
"grad_norm": 9.142548647265967,
|
|
"learning_rate": 3.189894058988528e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"epoch": 0.47065617555764094,
|
|
"grad_norm": 10.80278074451447,
|
|
"learning_rate": 3.183836681279344e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"epoch": 0.47137804085757595,
|
|
"grad_norm": 10.130379027496794,
|
|
"learning_rate": 3.177774961188524e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"epoch": 0.472099906157511,
|
|
"grad_norm": 7.542971934363328,
|
|
"learning_rate": 3.171708937208154e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"epoch": 0.472821771457446,
|
|
"grad_norm": 10.844193020610753,
|
|
"learning_rate": 3.1656386478576553e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"epoch": 0.4735436367573811,
|
|
"grad_norm": 8.465995887258453,
|
|
"learning_rate": 3.159564131683529e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"epoch": 0.4742655020573161,
|
|
"grad_norm": 10.357924748793268,
|
|
"learning_rate": 3.15348542725912e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1754,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"epoch": 0.4749873673572511,
|
|
"grad_norm": 8.386237735602531,
|
|
"learning_rate": 3.1474025731843675e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1686,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"epoch": 0.4757092326571862,
|
|
"grad_norm": 7.538386472518327,
|
|
"learning_rate": 3.141315608085561e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"epoch": 0.4764310979571212,
|
|
"grad_norm": 11.830534544208183,
|
|
"learning_rate": 3.1352245706150966e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.47715296325705625,
|
|
"grad_norm": 8.216481532924343,
|
|
"learning_rate": 3.129129499451229e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"epoch": 0.47787482855699126,
|
|
"grad_norm": 8.346007528728483,
|
|
"learning_rate": 3.123030433297823e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1709,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"epoch": 0.4785966938569263,
|
|
"grad_norm": 11.851428689841443,
|
|
"learning_rate": 3.11692741088412e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2184,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"epoch": 0.47931855915686133,
|
|
"grad_norm": 11.616947258566963,
|
|
"learning_rate": 3.110820470964479e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1676,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"epoch": 0.48004042445679634,
|
|
"grad_norm": 8.689762439600946,
|
|
"learning_rate": 3.1047096523181334e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"epoch": 0.4807622897567314,
|
|
"grad_norm": 11.54632495710895,
|
|
"learning_rate": 3.098594993748949e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1905,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"epoch": 0.4814841550566664,
|
|
"grad_norm": 8.85697591328895,
|
|
"learning_rate": 3.092476534085177e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1831,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"epoch": 0.4822060203566015,
|
|
"grad_norm": 6.837080063986226,
|
|
"learning_rate": 3.0863543121792023e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"epoch": 0.4829278856565365,
|
|
"grad_norm": 8.376719148936688,
|
|
"learning_rate": 3.080228366907302e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"epoch": 0.4836497509564715,
|
|
"grad_norm": 11.991657008786461,
|
|
"learning_rate": 3.074098737169395e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"epoch": 0.48437161625640657,
|
|
"grad_norm": 6.801458005281131,
|
|
"learning_rate": 3.0679654618887997e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"epoch": 0.4850934815563416,
|
|
"grad_norm": 9.454920867053952,
|
|
"learning_rate": 3.0618285800119807e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1882,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"epoch": 0.48581534685627664,
|
|
"grad_norm": 8.405586504954512,
|
|
"learning_rate": 3.0556881305083064e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1845,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"epoch": 0.48653721215621165,
|
|
"grad_norm": 9.286913760440065,
|
|
"learning_rate": 3.049544152369798e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1808,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"epoch": 0.48725907745614666,
|
|
"grad_norm": 9.84775376184747,
|
|
"learning_rate": 3.043396684610887e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"epoch": 0.4879809427560817,
|
|
"grad_norm": 9.876467151373804,
|
|
"learning_rate": 3.0372457662681597e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"epoch": 0.48870280805601674,
|
|
"grad_norm": 8.982483288644246,
|
|
"learning_rate": 3.031091436400117e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"epoch": 0.4894246733559518,
|
|
"grad_norm": 8.183235398587287,
|
|
"learning_rate": 3.024933734086922e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"epoch": 0.4901465386558868,
|
|
"grad_norm": 7.608116127645515,
|
|
"learning_rate": 3.018772698430152e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"epoch": 0.4908684039558218,
|
|
"grad_norm": 11.231827143842242,
|
|
"learning_rate": 3.0126083685525526e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1669,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.4915902692557569,
|
|
"grad_norm": 16.428828202202983,
|
|
"learning_rate": 3.006440783597787e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1696,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"epoch": 0.4923121345556919,
|
|
"grad_norm": 11.61196417270313,
|
|
"learning_rate": 3.000269982730189e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1766,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"epoch": 0.49303399985562696,
|
|
"grad_norm": 8.17788758978238,
|
|
"learning_rate": 2.9940960051345135e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"epoch": 0.49375586515556197,
|
|
"grad_norm": 9.07977507120006,
|
|
"learning_rate": 2.987918890015685e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"epoch": 0.494477730455497,
|
|
"grad_norm": 13.691630630266364,
|
|
"learning_rate": 2.981738676598555e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"epoch": 0.49519959575543204,
|
|
"grad_norm": 10.993076123812866,
|
|
"learning_rate": 2.9755554041276465e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1889,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"epoch": 0.49592146105536705,
|
|
"grad_norm": 11.646557564452921,
|
|
"learning_rate": 2.9693691118669117e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1677,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"epoch": 0.4966433263553021,
|
|
"grad_norm": 8.874925129899568,
|
|
"learning_rate": 2.9631798390994726e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1841,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"epoch": 0.4973651916552371,
|
|
"grad_norm": 8.267348409663557,
|
|
"learning_rate": 2.9569876251273826e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1684,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"epoch": 0.4980870569551722,
|
|
"grad_norm": 11.307205545557014,
|
|
"learning_rate": 2.9507925092713685e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1592,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"epoch": 0.4988089222551072,
|
|
"grad_norm": 9.259383366590928,
|
|
"learning_rate": 2.9445945308705866e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1956,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"epoch": 0.4995307875550422,
|
|
"grad_norm": 7.91332378148377,
|
|
"learning_rate": 2.938393729282369e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"epoch": 0.5002526528549772,
|
|
"grad_norm": 9.581050058932107,
|
|
"learning_rate": 2.932190143881976e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.165,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"epoch": 0.5009745181549123,
|
|
"grad_norm": 11.548026800858397,
|
|
"learning_rate": 2.925983814062342e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"epoch": 0.5016963834548473,
|
|
"grad_norm": 9.32142861680438,
|
|
"learning_rate": 2.9197747792338344e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1701,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"epoch": 0.5024182487547824,
|
|
"grad_norm": 8.751197433738259,
|
|
"learning_rate": 2.9135630788239936e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"epoch": 0.5031401140547174,
|
|
"grad_norm": 9.093507158463877,
|
|
"learning_rate": 2.907348752277286e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"epoch": 0.5038619793546524,
|
|
"grad_norm": 10.072538127620183,
|
|
"learning_rate": 2.901131839054856e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1771,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"epoch": 0.5045838446545875,
|
|
"grad_norm": 7.9225853596575515,
|
|
"learning_rate": 2.894912378634272e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"epoch": 0.5053057099545225,
|
|
"grad_norm": 8.508668273373068,
|
|
"learning_rate": 2.888690410509278e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1732,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.5060275752544575,
|
|
"grad_norm": 9.683917887834271,
|
|
"learning_rate": 2.8824659741895424e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"epoch": 0.5067494405543925,
|
|
"grad_norm": 11.888118286993855,
|
|
"learning_rate": 2.8762391092004054e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1856,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"epoch": 0.5074713058543275,
|
|
"grad_norm": 10.520962949102882,
|
|
"learning_rate": 2.8700098550826317e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1766,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"epoch": 0.5081931711542627,
|
|
"grad_norm": 6.050995114285996,
|
|
"learning_rate": 2.8637782513921537e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"epoch": 0.5089150364541977,
|
|
"grad_norm": 11.386068773907736,
|
|
"learning_rate": 2.857544337699826e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"epoch": 0.5096369017541327,
|
|
"grad_norm": 9.512039437649568,
|
|
"learning_rate": 2.85130815359117e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.17,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"epoch": 0.5103587670540677,
|
|
"grad_norm": 11.114854171347922,
|
|
"learning_rate": 2.845069738666128e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"epoch": 0.5110806323540027,
|
|
"grad_norm": 6.776138063327284,
|
|
"learning_rate": 2.8388291325388024e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"epoch": 0.5118024976539378,
|
|
"grad_norm": 9.638830345053352,
|
|
"learning_rate": 2.832586374837215e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1917,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"epoch": 0.5125243629538728,
|
|
"grad_norm": 10.274548246717263,
|
|
"learning_rate": 2.826341505203047e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"epoch": 0.5132462282538078,
|
|
"grad_norm": 10.65030499452255,
|
|
"learning_rate": 2.8200945632913917e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"epoch": 0.5139680935537428,
|
|
"grad_norm": 8.65595924084498,
|
|
"learning_rate": 2.8138455887705017e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1748,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"epoch": 0.5146899588536779,
|
|
"grad_norm": 7.416014752541617,
|
|
"learning_rate": 2.807594621321536e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"epoch": 0.515411824153613,
|
|
"grad_norm": 9.986965415691905,
|
|
"learning_rate": 2.801341700638307e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"epoch": 0.516133689453548,
|
|
"grad_norm": 8.63015173531394,
|
|
"learning_rate": 2.7950868664270355e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"epoch": 0.516855554753483,
|
|
"grad_norm": 8.92932402159572,
|
|
"learning_rate": 2.788830158406088e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"epoch": 0.517577420053418,
|
|
"grad_norm": 9.18548337260447,
|
|
"learning_rate": 2.782571616305731e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1761,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"epoch": 0.518299285353353,
|
|
"grad_norm": 7.559246854549426,
|
|
"learning_rate": 2.7763112798678787e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"epoch": 0.5190211506532881,
|
|
"grad_norm": 8.366664721309567,
|
|
"learning_rate": 2.77004918884584e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"epoch": 0.5197430159532231,
|
|
"grad_norm": 11.737242043765724,
|
|
"learning_rate": 2.763785383004063e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.5204648812531582,
|
|
"grad_norm": 7.926610565257705,
|
|
"learning_rate": 2.7575199021178855e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1644,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"epoch": 0.5211867465530932,
|
|
"grad_norm": 11.714571880510713,
|
|
"learning_rate": 2.751252785973284e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"epoch": 0.5219086118530282,
|
|
"grad_norm": 8.471661619696453,
|
|
"learning_rate": 2.744984074366617e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"epoch": 0.5226304771529633,
|
|
"grad_norm": 9.494843803969578,
|
|
"learning_rate": 2.738713807104375e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"epoch": 0.5233523424528983,
|
|
"grad_norm": 7.787106376583822,
|
|
"learning_rate": 2.732442024002926e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1461,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"epoch": 0.5240742077528333,
|
|
"grad_norm": 6.681800096835566,
|
|
"learning_rate": 2.726168764888264e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1681,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"epoch": 0.5247960730527683,
|
|
"grad_norm": 9.923678846098278,
|
|
"learning_rate": 2.7198940695957573e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"epoch": 0.5255179383527034,
|
|
"grad_norm": 9.08851776389925,
|
|
"learning_rate": 2.713617977969892e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.2118,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"epoch": 0.5262398036526384,
|
|
"grad_norm": 9.6683925726083,
|
|
"learning_rate": 2.707340529864022e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"epoch": 0.5269616689525735,
|
|
"grad_norm": 10.896898496114463,
|
|
"learning_rate": 2.7010617651401133e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"epoch": 0.5276835342525085,
|
|
"grad_norm": 10.726650561848246,
|
|
"learning_rate": 2.694781723668495e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"epoch": 0.5284053995524435,
|
|
"grad_norm": 10.009720787997963,
|
|
"learning_rate": 2.6885004453276014e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"epoch": 0.5291272648523786,
|
|
"grad_norm": 10.393937281174756,
|
|
"learning_rate": 2.682217970003724e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1809,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"epoch": 0.5298491301523136,
|
|
"grad_norm": 6.564234569234908,
|
|
"learning_rate": 2.6759343375907497e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1776,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"epoch": 0.5305709954522486,
|
|
"grad_norm": 8.875705075362786,
|
|
"learning_rate": 2.669649587989918e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"epoch": 0.5312928607521836,
|
|
"grad_norm": 8.149716947695198,
|
|
"learning_rate": 2.6633637611095615e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"epoch": 0.5320147260521186,
|
|
"grad_norm": 9.117623642609074,
|
|
"learning_rate": 2.6570768968648515e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"epoch": 0.5327365913520538,
|
|
"grad_norm": 7.7833308392028115,
|
|
"learning_rate": 2.6507890351775485e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"epoch": 0.5334584566519888,
|
|
"grad_norm": 11.143661735085152,
|
|
"learning_rate": 2.644500215975747e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.177,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"epoch": 0.5341803219519238,
|
|
"grad_norm": 6.014943903391089,
|
|
"learning_rate": 2.63821047919362e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1905,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.5349021872518588,
|
|
"grad_norm": 9.506926726417404,
|
|
"learning_rate": 2.631919864771168e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"epoch": 0.5356240525517938,
|
|
"grad_norm": 8.989178126574519,
|
|
"learning_rate": 2.6256284126539657e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"epoch": 0.5363459178517289,
|
|
"grad_norm": 8.096664424589033,
|
|
"learning_rate": 2.619336162792905e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.5,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"epoch": 0.5370677831516639,
|
|
"grad_norm": 16.841361704172662,
|
|
"learning_rate": 2.6130431551439456e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"epoch": 0.5377896484515989,
|
|
"grad_norm": 10.149973787308852,
|
|
"learning_rate": 2.606749429667859e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -788.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1571,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"epoch": 0.5385115137515339,
|
|
"grad_norm": 11.046710856292618,
|
|
"learning_rate": 2.6004550263299724e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1731,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"epoch": 0.539233379051469,
|
|
"grad_norm": 9.061807143540431,
|
|
"learning_rate": 2.5941599850999194e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -792.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1776,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -6.34375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"epoch": 0.5399552443514041,
|
|
"grad_norm": 10.136512372235272,
|
|
"learning_rate": 2.5878643459513857e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"epoch": 0.5406771096513391,
|
|
"grad_norm": 9.091121034585713,
|
|
"learning_rate": 2.58156814886185e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"epoch": 0.5413989749512741,
|
|
"grad_norm": 11.023486939102211,
|
|
"learning_rate": 2.575271433812338e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 0.5421208402512091,
|
|
"grad_norm": 12.378483126658342,
|
|
"learning_rate": 2.5689742407871615e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"epoch": 0.5428427055511441,
|
|
"grad_norm": 9.630319451714357,
|
|
"learning_rate": 2.562676609773669e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"epoch": 0.5435645708510792,
|
|
"grad_norm": 8.457889597977909,
|
|
"learning_rate": 2.556378580761989e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1797,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"epoch": 0.5442864361510142,
|
|
"grad_norm": 8.046649935179103,
|
|
"learning_rate": 2.5500801937447794e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1379,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"epoch": 0.5450083014509493,
|
|
"grad_norm": 9.046127207562416,
|
|
"learning_rate": 2.5437814887169684e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"epoch": 0.5457301667508843,
|
|
"grad_norm": 9.329772174624635,
|
|
"learning_rate": 2.5374825056755073e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"epoch": 0.5464520320508193,
|
|
"grad_norm": 9.9179656907273,
|
|
"learning_rate": 2.531183284619109e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"epoch": 0.5471738973507544,
|
|
"grad_norm": 9.557689966293122,
|
|
"learning_rate": 2.524883865548e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1632,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"epoch": 0.5478957626506894,
|
|
"grad_norm": 9.086340027521503,
|
|
"learning_rate": 2.5185842884636633e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"epoch": 0.5486176279506244,
|
|
"grad_norm": 8.606477368401597,
|
|
"learning_rate": 2.5122845933685873e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.5493394932505594,
|
|
"grad_norm": 9.684120993406607,
|
|
"learning_rate": 2.505984820266007e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"epoch": 0.5500613585504944,
|
|
"grad_norm": 11.673738089201613,
|
|
"learning_rate": 2.4996850091596535e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"epoch": 0.5507832238504295,
|
|
"grad_norm": 9.43384042742059,
|
|
"learning_rate": 2.493385200053502e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"epoch": 0.5515050891503646,
|
|
"grad_norm": 7.477187732924252,
|
|
"learning_rate": 2.48708543295151e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1651,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"epoch": 0.5522269544502996,
|
|
"grad_norm": 7.419622643126169,
|
|
"learning_rate": 2.480785747857372e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"epoch": 0.5529488197502346,
|
|
"grad_norm": 7.6418997353667955,
|
|
"learning_rate": 2.474486184774262e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"epoch": 0.5536706850501696,
|
|
"grad_norm": 11.143956805660984,
|
|
"learning_rate": 2.468186783704575e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"epoch": 0.5543925503501047,
|
|
"grad_norm": 12.296442116097905,
|
|
"learning_rate": 2.461887584649684e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"epoch": 0.5551144156500397,
|
|
"grad_norm": 13.050278426508344,
|
|
"learning_rate": 2.4555886276096713e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1799,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"epoch": 0.5558362809499747,
|
|
"grad_norm": 7.331592070155222,
|
|
"learning_rate": 2.4492899525830896e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"epoch": 0.5565581462499097,
|
|
"grad_norm": 10.969811602721066,
|
|
"learning_rate": 2.4429915995666967e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"epoch": 0.5572800115498447,
|
|
"grad_norm": 6.714447148185967,
|
|
"learning_rate": 2.4366936085552055e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1493,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"epoch": 0.5580018768497799,
|
|
"grad_norm": 6.283567486920552,
|
|
"learning_rate": 2.430396019541032e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"epoch": 0.5587237421497149,
|
|
"grad_norm": 9.641423565186832,
|
|
"learning_rate": 2.42409887251404e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"epoch": 0.5594456074496499,
|
|
"grad_norm": 9.82613619768625,
|
|
"learning_rate": 2.4178022074612834e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"epoch": 0.5601674727495849,
|
|
"grad_norm": 8.116478311651692,
|
|
"learning_rate": 2.4115060643667596e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"epoch": 0.56088933804952,
|
|
"grad_norm": 9.994907736669813,
|
|
"learning_rate": 2.4052104832111477e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"epoch": 0.561611203349455,
|
|
"grad_norm": 8.76711793301904,
|
|
"learning_rate": 2.39891550397156e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"epoch": 0.56233306864939,
|
|
"grad_norm": 8.295373017899776,
|
|
"learning_rate": 2.392621166621289e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1816,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"epoch": 0.563054933949325,
|
|
"grad_norm": 7.740744626757316,
|
|
"learning_rate": 2.386327511129548e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.56377679924926,
|
|
"grad_norm": 10.604529688792997,
|
|
"learning_rate": 2.380034577461221e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1593,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"epoch": 0.5644986645491952,
|
|
"grad_norm": 6.778093404198004,
|
|
"learning_rate": 2.373742405576611e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"epoch": 0.5652205298491302,
|
|
"grad_norm": 10.12858521011711,
|
|
"learning_rate": 2.3674510354311787e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1778,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7830
|
|
},
|
|
{
|
|
"epoch": 0.5659423951490652,
|
|
"grad_norm": 8.064195843692932,
|
|
"learning_rate": 2.3611605069752988e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"epoch": 0.5666642604490002,
|
|
"grad_norm": 9.775841021106938,
|
|
"learning_rate": 2.3548708601539982e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 7850
|
|
},
|
|
{
|
|
"epoch": 0.5673861257489352,
|
|
"grad_norm": 10.098938805871972,
|
|
"learning_rate": 2.3485821349067062e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"epoch": 0.5681079910488703,
|
|
"grad_norm": 8.30412143652236,
|
|
"learning_rate": 2.3422943711670015e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7870
|
|
},
|
|
{
|
|
"epoch": 0.5688298563488053,
|
|
"grad_norm": 6.817526291974242,
|
|
"learning_rate": 2.3360076088623544e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"epoch": 0.5695517216487404,
|
|
"grad_norm": 10.06504278640758,
|
|
"learning_rate": 2.3297218879138775e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1685,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7890
|
|
},
|
|
{
|
|
"epoch": 0.5702735869486754,
|
|
"grad_norm": 7.820665146752294,
|
|
"learning_rate": 2.3234372482360726e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"epoch": 0.5709954522486104,
|
|
"grad_norm": 11.261516761461985,
|
|
"learning_rate": 2.3171537297365718e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1524,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 7910
|
|
},
|
|
{
|
|
"epoch": 0.5717173175485455,
|
|
"grad_norm": 12.326978372880708,
|
|
"learning_rate": 2.3108713723158908e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1601,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"epoch": 0.5724391828484805,
|
|
"grad_norm": 12.561004648741765,
|
|
"learning_rate": 2.304590215867169e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7930
|
|
},
|
|
{
|
|
"epoch": 0.5731610481484155,
|
|
"grad_norm": 7.388599673863509,
|
|
"learning_rate": 2.298310300275924e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1454,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"epoch": 0.5738829134483505,
|
|
"grad_norm": 6.502079922036917,
|
|
"learning_rate": 2.2920316654197912e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7950
|
|
},
|
|
{
|
|
"epoch": 0.5746047787482855,
|
|
"grad_norm": 8.707306377676787,
|
|
"learning_rate": 2.2857543511682721e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"epoch": 0.5753266440482206,
|
|
"grad_norm": 9.94916607775227,
|
|
"learning_rate": 2.279478397382486e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 7970
|
|
},
|
|
{
|
|
"epoch": 0.5760485093481557,
|
|
"grad_norm": 6.43085308116789,
|
|
"learning_rate": 2.273203843914911e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1612,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"epoch": 0.5767703746480907,
|
|
"grad_norm": 9.384961354414472,
|
|
"learning_rate": 2.266930730609132e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1473,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 7990
|
|
},
|
|
{
|
|
"epoch": 0.5774922399480257,
|
|
"grad_norm": 7.869464671685872,
|
|
"learning_rate": 2.260659097299591e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 0.5782141052479607,
|
|
"grad_norm": 6.036220985959268,
|
|
"learning_rate": 2.2543889838113297e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1523,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 8010
|
|
},
|
|
{
|
|
"epoch": 0.5789359705478958,
|
|
"grad_norm": 9.091611380745027,
|
|
"learning_rate": 2.2481204299597422e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1708,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"epoch": 0.5796578358478308,
|
|
"grad_norm": 8.969271615418297,
|
|
"learning_rate": 2.2418534755503169e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.154,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8030
|
|
},
|
|
{
|
|
"epoch": 0.5803797011477658,
|
|
"grad_norm": 8.869611211845172,
|
|
"learning_rate": 2.235588160378384e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1892,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"epoch": 0.5811015664477008,
|
|
"grad_norm": 8.585563297437242,
|
|
"learning_rate": 2.229324524228868e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1972,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8050
|
|
},
|
|
{
|
|
"epoch": 0.5818234317476358,
|
|
"grad_norm": 6.361935798258764,
|
|
"learning_rate": 2.2230626068760297e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"epoch": 0.582545297047571,
|
|
"grad_norm": 7.123503913889777,
|
|
"learning_rate": 2.216802448083214e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1524,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8070
|
|
},
|
|
{
|
|
"epoch": 0.583267162347506,
|
|
"grad_norm": 5.3274420243546645,
|
|
"learning_rate": 2.2105440876026026e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"epoch": 0.583989027647441,
|
|
"grad_norm": 8.126969483825905,
|
|
"learning_rate": 2.2042875651749548e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1605,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8090
|
|
},
|
|
{
|
|
"epoch": 0.584710892947376,
|
|
"grad_norm": 6.82538345344394,
|
|
"learning_rate": 2.1980329205293585e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1633,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"epoch": 0.585432758247311,
|
|
"grad_norm": 12.200961629937073,
|
|
"learning_rate": 2.1917801933829807e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8110
|
|
},
|
|
{
|
|
"epoch": 0.5861546235472461,
|
|
"grad_norm": 7.906908172237448,
|
|
"learning_rate": 2.1855294234408068e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"epoch": 0.5868764888471811,
|
|
"grad_norm": 10.313898842262107,
|
|
"learning_rate": 2.1792806503953986e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8130
|
|
},
|
|
{
|
|
"epoch": 0.5875983541471161,
|
|
"grad_norm": 9.428346584688365,
|
|
"learning_rate": 2.1730339139266368e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"epoch": 0.5883202194470512,
|
|
"grad_norm": 9.348567817555846,
|
|
"learning_rate": 2.1667892537014664e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8150
|
|
},
|
|
{
|
|
"epoch": 0.5890420847469862,
|
|
"grad_norm": 9.578462386462538,
|
|
"learning_rate": 2.1605467093736534e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1611,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"epoch": 0.5897639500469213,
|
|
"grad_norm": 7.084598817360508,
|
|
"learning_rate": 2.1543063205835232e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1716,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8170
|
|
},
|
|
{
|
|
"epoch": 0.5904858153468563,
|
|
"grad_norm": 10.096964364918204,
|
|
"learning_rate": 2.1480681269577156e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"epoch": 0.5912076806467913,
|
|
"grad_norm": 13.036565026698682,
|
|
"learning_rate": 2.141832168108932e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8190
|
|
},
|
|
{
|
|
"epoch": 0.5919295459467263,
|
|
"grad_norm": 10.503733261465367,
|
|
"learning_rate": 2.1355984836356805e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1803,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"epoch": 0.5926514112466613,
|
|
"grad_norm": 8.837085032744609,
|
|
"learning_rate": 2.1293671131220277e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1517,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8210
|
|
},
|
|
{
|
|
"epoch": 0.5933732765465964,
|
|
"grad_norm": 13.492509568178333,
|
|
"learning_rate": 2.123138096137349e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1718,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"epoch": 0.5940951418465314,
|
|
"grad_norm": 4.422649076334775,
|
|
"learning_rate": 2.1169114722360708e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1322,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.5,
|
|
"step": 8230
|
|
},
|
|
{
|
|
"epoch": 0.5948170071464665,
|
|
"grad_norm": 8.207509821314797,
|
|
"learning_rate": 2.1106872809574266e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"epoch": 0.5955388724464015,
|
|
"grad_norm": 11.51646939268645,
|
|
"learning_rate": 2.1044655618251994e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8250
|
|
},
|
|
{
|
|
"epoch": 0.5962607377463366,
|
|
"grad_norm": 8.81738149975298,
|
|
"learning_rate": 2.098246354347477e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"epoch": 0.5969826030462716,
|
|
"grad_norm": 9.880728738377933,
|
|
"learning_rate": 2.0920296980163975e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8270
|
|
},
|
|
{
|
|
"epoch": 0.5977044683462066,
|
|
"grad_norm": 12.130445254705615,
|
|
"learning_rate": 2.0858156323078986e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"epoch": 0.5984263336461416,
|
|
"grad_norm": 9.18646238410699,
|
|
"learning_rate": 2.0796041966814658e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8290
|
|
},
|
|
{
|
|
"epoch": 0.5991481989460766,
|
|
"grad_norm": 7.4415035510148595,
|
|
"learning_rate": 2.0733954305798884e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1668,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"epoch": 0.5998700642460117,
|
|
"grad_norm": 9.921182858419193,
|
|
"learning_rate": 2.0671893734289984e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8310
|
|
},
|
|
{
|
|
"epoch": 0.6005919295459468,
|
|
"grad_norm": 6.738345944941375,
|
|
"learning_rate": 2.0609860646374305e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1682,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"epoch": 0.6013137948458818,
|
|
"grad_norm": 9.581111671762294,
|
|
"learning_rate": 2.054785543596363e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.174,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8330
|
|
},
|
|
{
|
|
"epoch": 0.6020356601458168,
|
|
"grad_norm": 9.266451111611559,
|
|
"learning_rate": 2.0485878496792758e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1594,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"epoch": 0.6027575254457518,
|
|
"grad_norm": 6.924514229924403,
|
|
"learning_rate": 2.042393022241695e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1331,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 8350
|
|
},
|
|
{
|
|
"epoch": 0.6034793907456869,
|
|
"grad_norm": 11.878033012527993,
|
|
"learning_rate": 2.0362011006209429e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"epoch": 0.6042012560456219,
|
|
"grad_norm": 6.456356558581018,
|
|
"learning_rate": 2.0300121241358925e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1412,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8370
|
|
},
|
|
{
|
|
"epoch": 0.6049231213455569,
|
|
"grad_norm": 6.2766000485447595,
|
|
"learning_rate": 2.023826132086714e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"epoch": 0.6056449866454919,
|
|
"grad_norm": 10.010107774455438,
|
|
"learning_rate": 2.0176431637546243e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8390
|
|
},
|
|
{
|
|
"epoch": 0.6063668519454269,
|
|
"grad_norm": 6.93155516769275,
|
|
"learning_rate": 2.0114632584016444e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1442,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"epoch": 0.6070887172453621,
|
|
"grad_norm": 9.44671796286649,
|
|
"learning_rate": 2.0052864552703392e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8410
|
|
},
|
|
{
|
|
"epoch": 0.6078105825452971,
|
|
"grad_norm": 10.256129872630872,
|
|
"learning_rate": 1.9991127935835796e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"epoch": 0.6085324478452321,
|
|
"grad_norm": 10.112281066079031,
|
|
"learning_rate": 1.9929423125442866e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8430
|
|
},
|
|
{
|
|
"epoch": 0.6092543131451671,
|
|
"grad_norm": 5.989996318215951,
|
|
"learning_rate": 1.9867750513351813e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1405,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"epoch": 0.6099761784451021,
|
|
"grad_norm": 7.127345044430176,
|
|
"learning_rate": 1.9806110491185434e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1401,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8450
|
|
},
|
|
{
|
|
"epoch": 0.6106980437450372,
|
|
"grad_norm": 8.882033575289205,
|
|
"learning_rate": 1.9744503450359555e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"epoch": 0.6114199090449722,
|
|
"grad_norm": 11.636675089992194,
|
|
"learning_rate": 1.968292978208055e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8470
|
|
},
|
|
{
|
|
"epoch": 0.6121417743449072,
|
|
"grad_norm": 4.708987187458577,
|
|
"learning_rate": 1.9621389877342926e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"epoch": 0.6128636396448423,
|
|
"grad_norm": 8.691036530455966,
|
|
"learning_rate": 1.9559884126926742e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8490
|
|
},
|
|
{
|
|
"epoch": 0.6135855049447773,
|
|
"grad_norm": 7.033771928824113,
|
|
"learning_rate": 1.9498412921395196e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"epoch": 0.6143073702447124,
|
|
"grad_norm": 6.420805831736498,
|
|
"learning_rate": 1.9436976651092142e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1517,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8510
|
|
},
|
|
{
|
|
"epoch": 0.6150292355446474,
|
|
"grad_norm": 9.804155572476585,
|
|
"learning_rate": 1.9375575706139557e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"epoch": 0.6157511008445824,
|
|
"grad_norm": 8.402725478167426,
|
|
"learning_rate": 1.931421047643513e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8530
|
|
},
|
|
{
|
|
"epoch": 0.6164729661445174,
|
|
"grad_norm": 7.928649293353565,
|
|
"learning_rate": 1.9252881351649764e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"epoch": 0.6171948314444524,
|
|
"grad_norm": 10.584642277815288,
|
|
"learning_rate": 1.919158872122505e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1709,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8550
|
|
},
|
|
{
|
|
"epoch": 0.6179166967443875,
|
|
"grad_norm": 5.528300128432007,
|
|
"learning_rate": 1.9130332974370888e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"epoch": 0.6186385620443225,
|
|
"grad_norm": 8.381790511594774,
|
|
"learning_rate": 1.9069114500062944e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8570
|
|
},
|
|
{
|
|
"epoch": 0.6193604273442576,
|
|
"grad_norm": 7.672005385585843,
|
|
"learning_rate": 1.9007933687040192e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1556,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"epoch": 0.6200822926441926,
|
|
"grad_norm": 7.6497315518223505,
|
|
"learning_rate": 1.8946790923802491e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8590
|
|
},
|
|
{
|
|
"epoch": 0.6208041579441276,
|
|
"grad_norm": 8.83669969757212,
|
|
"learning_rate": 1.8885686598608044e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"epoch": 0.6215260232440627,
|
|
"grad_norm": 11.726451803340803,
|
|
"learning_rate": 1.8824621099470986e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8610
|
|
},
|
|
{
|
|
"epoch": 0.6222478885439977,
|
|
"grad_norm": 6.926635938541544,
|
|
"learning_rate": 1.8763594814158926e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"epoch": 0.6229697538439327,
|
|
"grad_norm": 29.047689387473163,
|
|
"learning_rate": 1.8702608130190427e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8630
|
|
},
|
|
{
|
|
"epoch": 0.6236916191438677,
|
|
"grad_norm": 12.046850711595459,
|
|
"learning_rate": 1.8641661434832615e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"epoch": 0.6244134844438027,
|
|
"grad_norm": 13.389050315360532,
|
|
"learning_rate": 1.858075511509865e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1718,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8650
|
|
},
|
|
{
|
|
"epoch": 0.6251353497437379,
|
|
"grad_norm": 8.48405716596924,
|
|
"learning_rate": 1.8519889557745355e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"epoch": 0.6258572150436729,
|
|
"grad_norm": 7.718084904787592,
|
|
"learning_rate": 1.8459065149270675e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.204,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8670
|
|
},
|
|
{
|
|
"epoch": 0.6265790803436079,
|
|
"grad_norm": 8.645603679282008,
|
|
"learning_rate": 1.8398282275911265e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"epoch": 0.6273009456435429,
|
|
"grad_norm": 9.649146909089938,
|
|
"learning_rate": 1.833754132364003e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8690
|
|
},
|
|
{
|
|
"epoch": 0.6280228109434779,
|
|
"grad_norm": 10.143836034702232,
|
|
"learning_rate": 1.8276842678163694e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"epoch": 0.628744676243413,
|
|
"grad_norm": 6.224441882099418,
|
|
"learning_rate": 1.82161867249203e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 8710
|
|
},
|
|
{
|
|
"epoch": 0.629466541543348,
|
|
"grad_norm": 10.818946600627243,
|
|
"learning_rate": 1.8155573849076829e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"epoch": 0.630188406843283,
|
|
"grad_norm": 8.600830363109214,
|
|
"learning_rate": 1.8095004435526673e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8730
|
|
},
|
|
{
|
|
"epoch": 0.630910272143218,
|
|
"grad_norm": 7.278694825397051,
|
|
"learning_rate": 1.8034478868887293e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"epoch": 0.631632137443153,
|
|
"grad_norm": 10.061499794489972,
|
|
"learning_rate": 1.7973997533497684e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1721,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8750
|
|
},
|
|
{
|
|
"epoch": 0.6323540027430882,
|
|
"grad_norm": 12.377386224781029,
|
|
"learning_rate": 1.791356081341597e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.152,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"epoch": 0.6330758680430232,
|
|
"grad_norm": 7.666515466883803,
|
|
"learning_rate": 1.7853169092416992e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8770
|
|
},
|
|
{
|
|
"epoch": 0.6337977333429582,
|
|
"grad_norm": 9.2022093037141,
|
|
"learning_rate": 1.779282275398983e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"epoch": 0.6345195986428932,
|
|
"grad_norm": 11.875630627682433,
|
|
"learning_rate": 1.773252218133537e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 8790
|
|
},
|
|
{
|
|
"epoch": 0.6352414639428283,
|
|
"grad_norm": 12.286853044802243,
|
|
"learning_rate": 1.7672267757363917e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1767,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.21875,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"epoch": 0.6359633292427633,
|
|
"grad_norm": 7.782715805579312,
|
|
"learning_rate": 1.7612059864692696e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1434,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8810
|
|
},
|
|
{
|
|
"epoch": 0.6366851945426983,
|
|
"grad_norm": 10.338787396379217,
|
|
"learning_rate": 1.7551898885643478e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1478,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"epoch": 0.6374070598426334,
|
|
"grad_norm": 8.641921016163051,
|
|
"learning_rate": 1.749178520224014e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1519,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 8830
|
|
},
|
|
{
|
|
"epoch": 0.6381289251425684,
|
|
"grad_norm": 9.128585650717545,
|
|
"learning_rate": 1.7431719196206186e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"epoch": 0.6388507904425035,
|
|
"grad_norm": 11.694392580125832,
|
|
"learning_rate": 1.7371701248962426e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1925,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8850
|
|
},
|
|
{
|
|
"epoch": 0.6395726557424385,
|
|
"grad_norm": 8.29325313521127,
|
|
"learning_rate": 1.7311731741624458e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"epoch": 0.6402945210423735,
|
|
"grad_norm": 8.814547322060793,
|
|
"learning_rate": 1.7251811055000283e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1451,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8870
|
|
},
|
|
{
|
|
"epoch": 0.6410163863423085,
|
|
"grad_norm": 9.320265993143382,
|
|
"learning_rate": 1.7191939569587923e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.13,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"epoch": 0.6417382516422435,
|
|
"grad_norm": 10.838220312842498,
|
|
"learning_rate": 1.7132117665572932e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8890
|
|
},
|
|
{
|
|
"epoch": 0.6424601169421786,
|
|
"grad_norm": 6.55136471958019,
|
|
"learning_rate": 1.7072345722826035e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1315,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"epoch": 0.6431819822421136,
|
|
"grad_norm": 8.66688500340224,
|
|
"learning_rate": 1.701262412090072e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8910
|
|
},
|
|
{
|
|
"epoch": 0.6439038475420487,
|
|
"grad_norm": 9.629379415169119,
|
|
"learning_rate": 1.6952953239030779e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1416,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"epoch": 0.6446257128419837,
|
|
"grad_norm": 12.098110662139495,
|
|
"learning_rate": 1.6893333456127946e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 8930
|
|
},
|
|
{
|
|
"epoch": 0.6453475781419187,
|
|
"grad_norm": 8.256462573405791,
|
|
"learning_rate": 1.6833765150779484e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1609,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"epoch": 0.6460694434418538,
|
|
"grad_norm": 11.239144635721424,
|
|
"learning_rate": 1.677424870124575e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8950
|
|
},
|
|
{
|
|
"epoch": 0.6467913087417888,
|
|
"grad_norm": 8.418303487457777,
|
|
"learning_rate": 1.671478448545784e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.149,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"epoch": 0.6475131740417238,
|
|
"grad_norm": 9.492259339751383,
|
|
"learning_rate": 1.665537288101514e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8970
|
|
},
|
|
{
|
|
"epoch": 0.6482350393416588,
|
|
"grad_norm": 7.953989568068451,
|
|
"learning_rate": 1.6596014265182957e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1683,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"epoch": 0.6489569046415938,
|
|
"grad_norm": 7.299757937508117,
|
|
"learning_rate": 1.6536709014890147e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1577,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8990
|
|
},
|
|
{
|
|
"epoch": 0.649678769941529,
|
|
"grad_norm": 9.426751785884438,
|
|
"learning_rate": 1.6477457506726655e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 0.650400635241464,
|
|
"grad_norm": 10.774886673375532,
|
|
"learning_rate": 1.641826011694119e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9010
|
|
},
|
|
{
|
|
"epoch": 0.651122500541399,
|
|
"grad_norm": 9.624000604333325,
|
|
"learning_rate": 1.6359117221438784e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"epoch": 0.651844365841334,
|
|
"grad_norm": 9.17261641216877,
|
|
"learning_rate": 1.6300029195778453e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9030
|
|
},
|
|
{
|
|
"epoch": 0.652566231141269,
|
|
"grad_norm": 10.974103824549372,
|
|
"learning_rate": 1.624099641517078e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"epoch": 0.6532880964412041,
|
|
"grad_norm": 6.917901057511971,
|
|
"learning_rate": 1.6182019254475514e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1515,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9050
|
|
},
|
|
{
|
|
"epoch": 0.6540099617411391,
|
|
"grad_norm": 10.04014608485764,
|
|
"learning_rate": 1.6123098088199267e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1476,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"epoch": 0.6547318270410741,
|
|
"grad_norm": 11.307618522526226,
|
|
"learning_rate": 1.6064233290493048e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1568,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9070
|
|
},
|
|
{
|
|
"epoch": 0.6554536923410091,
|
|
"grad_norm": 11.567312788735324,
|
|
"learning_rate": 1.600542523514992e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"epoch": 0.6561755576409442,
|
|
"grad_norm": 6.469353343465349,
|
|
"learning_rate": 1.5946674295602668e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1644,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9090
|
|
},
|
|
{
|
|
"epoch": 0.6568974229408793,
|
|
"grad_norm": 9.939174768893757,
|
|
"learning_rate": 1.5887980844921338e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1646,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"epoch": 0.6576192882408143,
|
|
"grad_norm": 10.710746815227962,
|
|
"learning_rate": 1.5829345255810966e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9110
|
|
},
|
|
{
|
|
"epoch": 0.6583411535407493,
|
|
"grad_norm": 9.589016893008033,
|
|
"learning_rate": 1.5770767900609144e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1403,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"epoch": 0.6590630188406843,
|
|
"grad_norm": 10.535090311502815,
|
|
"learning_rate": 1.5712249151283674e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9130
|
|
},
|
|
{
|
|
"epoch": 0.6597848841406193,
|
|
"grad_norm": 11.07484592479438,
|
|
"learning_rate": 1.565378937943022e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"epoch": 0.6605067494405544,
|
|
"grad_norm": 8.804459530019093,
|
|
"learning_rate": 1.559538895626994e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9150
|
|
},
|
|
{
|
|
"epoch": 0.6612286147404894,
|
|
"grad_norm": 9.869851207701721,
|
|
"learning_rate": 1.5537048252647102e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"epoch": 0.6619504800404244,
|
|
"grad_norm": 7.227577491074705,
|
|
"learning_rate": 1.547876763902679e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1604,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9170
|
|
},
|
|
{
|
|
"epoch": 0.6626723453403595,
|
|
"grad_norm": 8.512681174359171,
|
|
"learning_rate": 1.5420547485492483e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"epoch": 0.6633942106402945,
|
|
"grad_norm": 6.940912396524942,
|
|
"learning_rate": 1.5362388161743743e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9190
|
|
},
|
|
{
|
|
"epoch": 0.6641160759402296,
|
|
"grad_norm": 7.482638973278819,
|
|
"learning_rate": 1.5304290037093887e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"epoch": 0.6648379412401646,
|
|
"grad_norm": 9.239067837458254,
|
|
"learning_rate": 1.524625348046758e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1324,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9210
|
|
},
|
|
{
|
|
"epoch": 0.6655598065400996,
|
|
"grad_norm": 10.312943276531238,
|
|
"learning_rate": 1.5188278860398543e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1548,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"epoch": 0.6662816718400346,
|
|
"grad_norm": 12.377500015344213,
|
|
"learning_rate": 1.5130366545027215e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9230
|
|
},
|
|
{
|
|
"epoch": 0.6670035371399696,
|
|
"grad_norm": 11.437994745400522,
|
|
"learning_rate": 1.507251690209837e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"epoch": 0.6677254024399047,
|
|
"grad_norm": 9.227052828667116,
|
|
"learning_rate": 1.5014730298958834e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1733,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9250
|
|
},
|
|
{
|
|
"epoch": 0.6684472677398398,
|
|
"grad_norm": 11.34701390652657,
|
|
"learning_rate": 1.4957007102555101e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"epoch": 0.6691691330397748,
|
|
"grad_norm": 10.840533844995983,
|
|
"learning_rate": 1.489934767943106e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9270
|
|
},
|
|
{
|
|
"epoch": 0.6698909983397098,
|
|
"grad_norm": 5.005020117833695,
|
|
"learning_rate": 1.484175239572563e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"epoch": 0.6706128636396449,
|
|
"grad_norm": 9.438942465976393,
|
|
"learning_rate": 1.4784221617170427e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1544,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9290
|
|
},
|
|
{
|
|
"epoch": 0.6713347289395799,
|
|
"grad_norm": 8.775509056317704,
|
|
"learning_rate": 1.4726755709087467e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1568,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"epoch": 0.6720565942395149,
|
|
"grad_norm": 12.815148122117906,
|
|
"learning_rate": 1.466935503638686e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1535,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9310
|
|
},
|
|
{
|
|
"epoch": 0.6727784595394499,
|
|
"grad_norm": 6.322537492099079,
|
|
"learning_rate": 1.4612019963564425e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"epoch": 0.6735003248393849,
|
|
"grad_norm": 8.684754690608713,
|
|
"learning_rate": 1.4554750854699454e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 9330
|
|
},
|
|
{
|
|
"epoch": 0.67422219013932,
|
|
"grad_norm": 7.495450070953094,
|
|
"learning_rate": 1.4497548073452353e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"epoch": 0.6749440554392551,
|
|
"grad_norm": 9.818417235804333,
|
|
"learning_rate": 1.444041198306235e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9350
|
|
},
|
|
{
|
|
"epoch": 0.6756659207391901,
|
|
"grad_norm": 10.0125757389507,
|
|
"learning_rate": 1.4383342946345178e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"epoch": 0.6763877860391251,
|
|
"grad_norm": 10.056618279215506,
|
|
"learning_rate": 1.432634132569079e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9370
|
|
},
|
|
{
|
|
"epoch": 0.6771096513390601,
|
|
"grad_norm": 12.261509335918284,
|
|
"learning_rate": 1.4269407483061025e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"epoch": 0.6778315166389952,
|
|
"grad_norm": 9.583901573024836,
|
|
"learning_rate": 1.4212541779987358e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1808,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9390
|
|
},
|
|
{
|
|
"epoch": 0.6785533819389302,
|
|
"grad_norm": 7.602147425073978,
|
|
"learning_rate": 1.415574457756853e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1429,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"epoch": 0.6792752472388652,
|
|
"grad_norm": 5.715088913700748,
|
|
"learning_rate": 1.409901623646837e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9410
|
|
},
|
|
{
|
|
"epoch": 0.6799971125388002,
|
|
"grad_norm": 7.623897333532847,
|
|
"learning_rate": 1.4042357116913365e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"epoch": 0.6807189778387353,
|
|
"grad_norm": 12.781776965070957,
|
|
"learning_rate": 1.3985767578690474e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1763,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9430
|
|
},
|
|
{
|
|
"epoch": 0.6814408431386704,
|
|
"grad_norm": 10.237854404923459,
|
|
"learning_rate": 1.3929247981144845e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"epoch": 0.6821627084386054,
|
|
"grad_norm": 9.894822207763886,
|
|
"learning_rate": 1.387279868317744e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9450
|
|
},
|
|
{
|
|
"epoch": 0.6828845737385404,
|
|
"grad_norm": 9.241271465537533,
|
|
"learning_rate": 1.3816420043242843e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"epoch": 0.6836064390384754,
|
|
"grad_norm": 10.237589705127691,
|
|
"learning_rate": 1.3760112419346986e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9470
|
|
},
|
|
{
|
|
"epoch": 0.6843283043384104,
|
|
"grad_norm": 10.725962046372151,
|
|
"learning_rate": 1.3703876169044802e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"epoch": 0.6850501696383455,
|
|
"grad_norm": 7.6917366331039325,
|
|
"learning_rate": 1.364771164943802e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9490
|
|
},
|
|
{
|
|
"epoch": 0.6857720349382805,
|
|
"grad_norm": 8.728014646166761,
|
|
"learning_rate": 1.3591619217172883e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1455,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"epoch": 0.6864939002382155,
|
|
"grad_norm": 10.677477412212683,
|
|
"learning_rate": 1.3535599228437867e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1626,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9510
|
|
},
|
|
{
|
|
"epoch": 0.6872157655381506,
|
|
"grad_norm": 9.38419202809167,
|
|
"learning_rate": 1.3479652038961432e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"epoch": 0.6879376308380856,
|
|
"grad_norm": 8.766505031785163,
|
|
"learning_rate": 1.3423778004009762e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9530
|
|
},
|
|
{
|
|
"epoch": 0.6886594961380207,
|
|
"grad_norm": 7.070381554572682,
|
|
"learning_rate": 1.3367977478384513e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"epoch": 0.6893813614379557,
|
|
"grad_norm": 13.363793450146552,
|
|
"learning_rate": 1.3312250816420542e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1678,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9550
|
|
},
|
|
{
|
|
"epoch": 0.6901032267378907,
|
|
"grad_norm": 6.54534750827785,
|
|
"learning_rate": 1.3256598371983686e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"epoch": 0.6908250920378257,
|
|
"grad_norm": 8.065890871247143,
|
|
"learning_rate": 1.320102049846849e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1447,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9570
|
|
},
|
|
{
|
|
"epoch": 0.6915469573377607,
|
|
"grad_norm": 8.464118508817007,
|
|
"learning_rate": 1.314551754879595e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1415,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"epoch": 0.6922688226376958,
|
|
"grad_norm": 9.854707772376594,
|
|
"learning_rate": 1.3090089875411337e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.129,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 9590
|
|
},
|
|
{
|
|
"epoch": 0.6929906879376309,
|
|
"grad_norm": 8.672649571433473,
|
|
"learning_rate": 1.30347378302819e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"epoch": 0.6937125532375659,
|
|
"grad_norm": 9.034737955151657,
|
|
"learning_rate": 1.2979461764894608e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9610
|
|
},
|
|
{
|
|
"epoch": 0.6944344185375009,
|
|
"grad_norm": 8.90041609096791,
|
|
"learning_rate": 1.2924262030254022e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"epoch": 0.6951562838374359,
|
|
"grad_norm": 8.665967459471112,
|
|
"learning_rate": 1.2869138976879961e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9630
|
|
},
|
|
{
|
|
"epoch": 0.695878149137371,
|
|
"grad_norm": 7.497535290815869,
|
|
"learning_rate": 1.2814092954805294e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.171,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"epoch": 0.696600014437306,
|
|
"grad_norm": 9.990492914302166,
|
|
"learning_rate": 1.275912431357381e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9650
|
|
},
|
|
{
|
|
"epoch": 0.697321879737241,
|
|
"grad_norm": 7.823463866960965,
|
|
"learning_rate": 1.2704233402237858e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1707,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"epoch": 0.698043745037176,
|
|
"grad_norm": 7.579915303042644,
|
|
"learning_rate": 1.2649420569356217e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9670
|
|
},
|
|
{
|
|
"epoch": 0.698765610337111,
|
|
"grad_norm": 8.800622349499235,
|
|
"learning_rate": 1.2594686162991914e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1402,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"epoch": 0.6994874756370462,
|
|
"grad_norm": 9.327395207500006,
|
|
"learning_rate": 1.2540030530709888e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9690
|
|
},
|
|
{
|
|
"epoch": 0.7002093409369812,
|
|
"grad_norm": 8.376068930149511,
|
|
"learning_rate": 1.248545401957492e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1543,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"epoch": 0.7009312062369162,
|
|
"grad_norm": 7.936678196185944,
|
|
"learning_rate": 1.2430956976149345e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9710
|
|
},
|
|
{
|
|
"epoch": 0.7016530715368512,
|
|
"grad_norm": 12.345046600610592,
|
|
"learning_rate": 1.2376539746490878e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1713,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"epoch": 0.7023749368367862,
|
|
"grad_norm": 9.162811933963393,
|
|
"learning_rate": 1.2322202676150417e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9730
|
|
},
|
|
{
|
|
"epoch": 0.7030968021367213,
|
|
"grad_norm": 10.256111775264358,
|
|
"learning_rate": 1.2267946110169847e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"epoch": 0.7038186674366563,
|
|
"grad_norm": 8.584246074026865,
|
|
"learning_rate": 1.2213770393079846e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9750
|
|
},
|
|
{
|
|
"epoch": 0.7045405327365913,
|
|
"grad_norm": 7.583949089234293,
|
|
"learning_rate": 1.21596758688977e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1512,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"epoch": 0.7052623980365264,
|
|
"grad_norm": 9.44743736005493,
|
|
"learning_rate": 1.2105662881125112e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1652,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9770
|
|
},
|
|
{
|
|
"epoch": 0.7059842633364615,
|
|
"grad_norm": 6.4078324266348705,
|
|
"learning_rate": 1.205173177274604e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"epoch": 0.7067061286363965,
|
|
"grad_norm": 7.907790182080389,
|
|
"learning_rate": 1.1997882886224497e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1424,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9790
|
|
},
|
|
{
|
|
"epoch": 0.7074279939363315,
|
|
"grad_norm": 12.95788224606365,
|
|
"learning_rate": 1.194411656350238e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1558,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"epoch": 0.7081498592362665,
|
|
"grad_norm": 9.166806155356193,
|
|
"learning_rate": 1.1890433145997327e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1731,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9810
|
|
},
|
|
{
|
|
"epoch": 0.7088717245362015,
|
|
"grad_norm": 9.72249270328465,
|
|
"learning_rate": 1.1836832974600483e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"epoch": 0.7095935898361366,
|
|
"grad_norm": 7.965526025357869,
|
|
"learning_rate": 1.1783316389674406e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9830
|
|
},
|
|
{
|
|
"epoch": 0.7103154551360716,
|
|
"grad_norm": 7.929357398986534,
|
|
"learning_rate": 1.1729883731050899e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1315,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"epoch": 0.7110373204360066,
|
|
"grad_norm": 8.634524210955284,
|
|
"learning_rate": 1.1676535338028781e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9850
|
|
},
|
|
{
|
|
"epoch": 0.7117591857359417,
|
|
"grad_norm": 9.065771203848671,
|
|
"learning_rate": 1.1623271549371808e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1781,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"epoch": 0.7124810510358767,
|
|
"grad_norm": 7.949345789146648,
|
|
"learning_rate": 1.1570092703306523e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9870
|
|
},
|
|
{
|
|
"epoch": 0.7132029163358118,
|
|
"grad_norm": 8.559039194521684,
|
|
"learning_rate": 1.1516999137520023e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"epoch": 0.7139247816357468,
|
|
"grad_norm": 8.830447748470409,
|
|
"learning_rate": 1.1463991189157917e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1701,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9890
|
|
},
|
|
{
|
|
"epoch": 0.7146466469356818,
|
|
"grad_norm": 7.4835869108530275,
|
|
"learning_rate": 1.1411069194822124e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"epoch": 0.7153685122356168,
|
|
"grad_norm": 12.056792194890452,
|
|
"learning_rate": 1.1358233490568758e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1486,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9910
|
|
},
|
|
{
|
|
"epoch": 0.7160903775355518,
|
|
"grad_norm": 7.752875524749948,
|
|
"learning_rate": 1.1305484411905986e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1407,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"epoch": 0.716812242835487,
|
|
"grad_norm": 7.773726914737785,
|
|
"learning_rate": 1.12528222937919e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9930
|
|
},
|
|
{
|
|
"epoch": 0.717534108135422,
|
|
"grad_norm": 10.91086916613174,
|
|
"learning_rate": 1.1200247470632392e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -784.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1779,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.25,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"epoch": 0.718255973435357,
|
|
"grad_norm": 8.136219531246716,
|
|
"learning_rate": 1.1147760276279028e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9950
|
|
},
|
|
{
|
|
"epoch": 0.718977838735292,
|
|
"grad_norm": 12.213084226027048,
|
|
"learning_rate": 1.1095361044026927e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"epoch": 0.719699704035227,
|
|
"grad_norm": 7.447846242333707,
|
|
"learning_rate": 1.1043050106612656e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9970
|
|
},
|
|
{
|
|
"epoch": 0.7204215693351621,
|
|
"grad_norm": 11.497519578105333,
|
|
"learning_rate": 1.0990827796212074e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"epoch": 0.7211434346350971,
|
|
"grad_norm": 9.931460815710798,
|
|
"learning_rate": 1.0938694444438307e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9990
|
|
},
|
|
{
|
|
"epoch": 0.7218652999350321,
|
|
"grad_norm": 9.875757410099292,
|
|
"learning_rate": 1.0886650382339566e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 0.7218652999350321,
|
|
"eval_logits/chosen": -2.9375,
|
|
"eval_logits/rejected": -2.625,
|
|
"eval_logps/chosen": -772.0,
|
|
"eval_logps/rejected": -1168.0,
|
|
"eval_loss": 0.21195410192012787,
|
|
"eval_rewards/accuracies": 0.9136363863945007,
|
|
"eval_rewards/chosen": -6.0,
|
|
"eval_rewards/margins": 4.1875,
|
|
"eval_rewards/rejected": -10.1875,
|
|
"eval_runtime": 3594.9603,
|
|
"eval_samples_per_second": 27.4,
|
|
"eval_steps_per_second": 0.428,
|
|
"step": 10000
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 13853,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 5000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|