2924 lines
101 KiB
JSON
2924 lines
101 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 318.0,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -1.7139909267425537,
|
|
"logits/rejected": -1.6882292032241821,
|
|
"logps/chosen": -0.27932801842689514,
|
|
"logps/rejected": -0.2822762131690979,
|
|
"loss": 97.24929809570312,
|
|
"loss/core": 97.24929809570312,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.420787811279297,
|
|
"rewards/margins": 1.4634535312652588,
|
|
"rewards/rejected": 0.9573341608047485,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 2240.0,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -1.9642683267593384,
|
|
"logits/rejected": -2.0518295764923096,
|
|
"logps/chosen": -0.28963160514831543,
|
|
"logps/rejected": -0.28823795914649963,
|
|
"loss": 99.95600128173828,
|
|
"loss/core": 99.95600128173828,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6873565912246704,
|
|
"rewards/margins": 0.07187187671661377,
|
|
"rewards/rejected": 1.615484595298767,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 296.0,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -1.79877507686615,
|
|
"logits/rejected": -1.7281420230865479,
|
|
"logps/chosen": -0.27656567096710205,
|
|
"logps/rejected": -0.2845936417579651,
|
|
"loss": 98.14198303222656,
|
|
"loss/core": 98.14198303222656,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1426477432250977,
|
|
"rewards/margins": 1.0079020261764526,
|
|
"rewards/rejected": 1.134745478630066,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 232.0,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -1.9977076053619385,
|
|
"logits/rejected": -1.9979604482650757,
|
|
"logps/chosen": -0.30147844552993774,
|
|
"logps/rejected": -0.28688931465148926,
|
|
"loss": 98.10304260253906,
|
|
"loss/core": 98.10304260253906,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9145663976669312,
|
|
"rewards/margins": 1.0382013320922852,
|
|
"rewards/rejected": 0.876365065574646,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 184.0,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -1.9077049493789673,
|
|
"logits/rejected": -1.9242079257965088,
|
|
"logps/chosen": -0.2705998420715332,
|
|
"logps/rejected": -0.25868508219718933,
|
|
"loss": 96.9577407836914,
|
|
"loss/core": 96.9577407836914,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.849167585372925,
|
|
"rewards/margins": 2.2114977836608887,
|
|
"rewards/rejected": 1.637669324874878,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 288.0,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.088418483734131,
|
|
"logits/rejected": -2.0663094520568848,
|
|
"logps/chosen": -0.30147606134414673,
|
|
"logps/rejected": -0.3242497146129608,
|
|
"loss": 97.91767883300781,
|
|
"loss/core": 97.91767883300781,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.264533758163452,
|
|
"rewards/margins": 1.2376611232757568,
|
|
"rewards/rejected": 1.0268726348876953,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 103.0,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -1.867283821105957,
|
|
"logits/rejected": -1.8813632726669312,
|
|
"logps/chosen": -0.26848095655441284,
|
|
"logps/rejected": -0.26555073261260986,
|
|
"loss": 97.41613006591797,
|
|
"loss/core": 97.41613006591797,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6509482860565186,
|
|
"rewards/margins": 1.494160771369934,
|
|
"rewards/rejected": 1.1567875146865845,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 326.0,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -1.6814746856689453,
|
|
"logits/rejected": -1.7323602437973022,
|
|
"logps/chosen": -0.2946423888206482,
|
|
"logps/rejected": -0.2834400236606598,
|
|
"loss": 95.7684097290039,
|
|
"loss/core": 95.7684097290039,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.45792293548584,
|
|
"rewards/margins": 2.8708791732788086,
|
|
"rewards/rejected": 1.5870436429977417,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -1.8644527196884155,
|
|
"logits/rejected": -1.9930543899536133,
|
|
"logps/chosen": -0.2883604168891907,
|
|
"logps/rejected": -0.28879010677337646,
|
|
"loss": 97.59373474121094,
|
|
"loss/core": 97.59373474121094,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.0999531745910645,
|
|
"rewards/margins": 1.261714220046997,
|
|
"rewards/rejected": 0.8382388949394226,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 812.0,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -1.9386039972305298,
|
|
"logits/rejected": -1.9440422058105469,
|
|
"logps/chosen": -0.2817879319190979,
|
|
"logps/rejected": -0.2857362627983093,
|
|
"loss": 97.58670043945312,
|
|
"loss/core": 97.58670043945312,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8786211013793945,
|
|
"rewards/margins": 1.4362843036651611,
|
|
"rewards/rejected": 1.4423367977142334,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 62.75,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -1.906114935874939,
|
|
"logits/rejected": -1.8699897527694702,
|
|
"logps/chosen": -0.2895704507827759,
|
|
"logps/rejected": -0.2879449427127838,
|
|
"loss": 100.38077545166016,
|
|
"loss/core": 100.38077545166016,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2526936531066895,
|
|
"rewards/margins": 0.2057073414325714,
|
|
"rewards/rejected": 2.0469868183135986,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 3264.0,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -1.828805923461914,
|
|
"logits/rejected": -1.8161197900772095,
|
|
"logps/chosen": -0.2744297683238983,
|
|
"logps/rejected": -0.28596776723861694,
|
|
"loss": 98.89493560791016,
|
|
"loss/core": 98.89493560791016,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.156543731689453,
|
|
"rewards/margins": 0.6224328279495239,
|
|
"rewards/rejected": 1.5341110229492188,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -1.6605703830718994,
|
|
"logits/rejected": -1.72246515750885,
|
|
"logps/chosen": -0.3099282383918762,
|
|
"logps/rejected": -0.3059017062187195,
|
|
"loss": 96.74524688720703,
|
|
"loss/core": 96.74524688720703,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9625723361968994,
|
|
"rewards/margins": 1.7866805791854858,
|
|
"rewards/rejected": 1.175891637802124,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 442.0,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.65080988407135,
|
|
"logits/rejected": -1.679664969444275,
|
|
"logps/chosen": -0.3835355043411255,
|
|
"logps/rejected": -0.2955405116081238,
|
|
"loss": 98.49574279785156,
|
|
"loss/core": 98.49574279785156,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6017367839813232,
|
|
"rewards/margins": 1.4039216041564941,
|
|
"rewards/rejected": 2.197815418243408,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 102.5,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -1.711356520652771,
|
|
"logits/rejected": -1.7348101139068604,
|
|
"logps/chosen": -0.3151581585407257,
|
|
"logps/rejected": -0.2897110879421234,
|
|
"loss": 98.30010223388672,
|
|
"loss/core": 98.30010223388672,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6052944660186768,
|
|
"rewards/margins": 1.0178263187408447,
|
|
"rewards/rejected": 1.5874683856964111,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 112.0,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -1.752758264541626,
|
|
"logits/rejected": -1.8684453964233398,
|
|
"logps/chosen": -0.26047635078430176,
|
|
"logps/rejected": -0.27302050590515137,
|
|
"loss": 96.66983795166016,
|
|
"loss/core": 96.66983795166016,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.242232084274292,
|
|
"rewards/margins": 1.980281114578247,
|
|
"rewards/rejected": 1.2619515657424927,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 159.0,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -1.8509635925292969,
|
|
"logits/rejected": -1.8500969409942627,
|
|
"logps/chosen": -0.3067242503166199,
|
|
"logps/rejected": -0.27861541509628296,
|
|
"loss": 96.42810821533203,
|
|
"loss/core": 96.42810821533203,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.5060653686523438,
|
|
"rewards/margins": 2.184356212615967,
|
|
"rewards/rejected": 1.321709394454956,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 64.5,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.0346593856811523,
|
|
"logits/rejected": -2.0443930625915527,
|
|
"logps/chosen": -0.28125816583633423,
|
|
"logps/rejected": -0.29576218128204346,
|
|
"loss": 98.27728271484375,
|
|
"loss/core": 98.27728271484375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7548061609268188,
|
|
"rewards/margins": 0.9071924090385437,
|
|
"rewards/rejected": 0.8476136922836304,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 276.0,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -1.6161243915557861,
|
|
"logits/rejected": -1.6432390213012695,
|
|
"logps/chosen": -0.26751023530960083,
|
|
"logps/rejected": -0.2801710367202759,
|
|
"loss": 98.1581039428711,
|
|
"loss/core": 98.1581039428711,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2086246013641357,
|
|
"rewards/margins": 1.1166317462921143,
|
|
"rewards/rejected": 2.0919928550720215,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -1.895495057106018,
|
|
"logits/rejected": -1.9128100872039795,
|
|
"logps/chosen": -0.25704631209373474,
|
|
"logps/rejected": -0.24669785797595978,
|
|
"loss": 97.79266357421875,
|
|
"loss/core": 97.79266357421875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.4641220569610596,
|
|
"rewards/margins": 1.1781212091445923,
|
|
"rewards/rejected": 1.2860008478164673,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 388.0,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -1.9339812994003296,
|
|
"logits/rejected": -1.9512369632720947,
|
|
"logps/chosen": -0.28484174609184265,
|
|
"logps/rejected": -0.28055232763290405,
|
|
"loss": 97.86317443847656,
|
|
"loss/core": 97.86317443847656,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9908250570297241,
|
|
"rewards/margins": 1.2003737688064575,
|
|
"rewards/rejected": 0.7904513478279114,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 149.0,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -1.73562753200531,
|
|
"logits/rejected": -1.6282856464385986,
|
|
"logps/chosen": -0.34042173624038696,
|
|
"logps/rejected": -0.2966521084308624,
|
|
"loss": 95.82826232910156,
|
|
"loss/core": 95.82826232910156,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.451060771942139,
|
|
"rewards/margins": 2.5516746044158936,
|
|
"rewards/rejected": 1.8993858098983765,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 696.0,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -1.6403529644012451,
|
|
"logits/rejected": -1.5681908130645752,
|
|
"logps/chosen": -0.3004806637763977,
|
|
"logps/rejected": -0.2985830008983612,
|
|
"loss": 99.58712005615234,
|
|
"loss/core": 99.58712005615234,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7745640277862549,
|
|
"rewards/margins": 0.32534095644950867,
|
|
"rewards/rejected": 1.4492229223251343,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 1528.0,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -1.6866496801376343,
|
|
"logits/rejected": -1.690090537071228,
|
|
"logps/chosen": -0.3020043969154358,
|
|
"logps/rejected": -0.29816195368766785,
|
|
"loss": 98.63155364990234,
|
|
"loss/core": 98.63155364990234,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.811497449874878,
|
|
"rewards/margins": 0.7096825838088989,
|
|
"rewards/rejected": 1.101815104484558,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 103.0,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -1.8087574243545532,
|
|
"logits/rejected": -1.8217788934707642,
|
|
"logps/chosen": -0.2990450859069824,
|
|
"logps/rejected": -0.306710809469223,
|
|
"loss": 98.02079772949219,
|
|
"loss/core": 98.02079772949219,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0650312900543213,
|
|
"rewards/margins": 1.0487768650054932,
|
|
"rewards/rejected": 1.0162544250488281,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 215.0,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -1.6154258251190186,
|
|
"logits/rejected": -1.73139226436615,
|
|
"logps/chosen": -0.27999377250671387,
|
|
"logps/rejected": -0.278720498085022,
|
|
"loss": 98.19866943359375,
|
|
"loss/core": 98.19866943359375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8072621822357178,
|
|
"rewards/margins": 0.9196640849113464,
|
|
"rewards/rejected": 0.8875982165336609,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 298.0,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -1.7737205028533936,
|
|
"logits/rejected": -1.8461641073226929,
|
|
"logps/chosen": -0.3029175102710724,
|
|
"logps/rejected": -0.2972210645675659,
|
|
"loss": 96.26443481445312,
|
|
"loss/core": 96.26443481445312,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3659565448760986,
|
|
"rewards/margins": 2.0419297218322754,
|
|
"rewards/rejected": 1.3240267038345337,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.091590404510498,
|
|
"logits/rejected": -2.117504596710205,
|
|
"logps/chosen": -0.2827909588813782,
|
|
"logps/rejected": -0.2891230285167694,
|
|
"loss": 98.2292251586914,
|
|
"loss/core": 98.2292251586914,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.625351905822754,
|
|
"rewards/margins": 0.9071060419082642,
|
|
"rewards/rejected": 0.7182460427284241,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 148.0,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.560279130935669,
|
|
"logits/rejected": -1.5604007244110107,
|
|
"logps/chosen": -0.30558672547340393,
|
|
"logps/rejected": -0.2993968725204468,
|
|
"loss": 96.93885040283203,
|
|
"loss/core": 96.93885040283203,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.561701536178589,
|
|
"rewards/margins": 1.752966284751892,
|
|
"rewards/rejected": 1.8087351322174072,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 256.0,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -1.9131495952606201,
|
|
"logits/rejected": -1.9743051528930664,
|
|
"logps/chosen": -0.3162950873374939,
|
|
"logps/rejected": -0.30938461422920227,
|
|
"loss": 96.15104675292969,
|
|
"loss/core": 96.15104675292969,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.649721622467041,
|
|
"rewards/margins": 2.2039794921875,
|
|
"rewards/rejected": 1.4457422494888306,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -1.809692144393921,
|
|
"logits/rejected": -1.869560956954956,
|
|
"logps/chosen": -0.2845507562160492,
|
|
"logps/rejected": -0.28922533988952637,
|
|
"loss": 96.42918395996094,
|
|
"loss/core": 96.42918395996094,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.84999418258667,
|
|
"rewards/margins": 1.9925158023834229,
|
|
"rewards/rejected": 0.8574784994125366,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 556.0,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -1.7576831579208374,
|
|
"logits/rejected": -1.7200205326080322,
|
|
"logps/chosen": -0.31108418107032776,
|
|
"logps/rejected": -0.31466466188430786,
|
|
"loss": 97.29283905029297,
|
|
"loss/core": 97.29283905029297,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.1445982456207275,
|
|
"rewards/margins": 1.7368183135986328,
|
|
"rewards/rejected": 1.4077800512313843,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -1.7794075012207031,
|
|
"logits/rejected": -1.7277469635009766,
|
|
"logps/chosen": -0.31061312556266785,
|
|
"logps/rejected": -0.3058010935783386,
|
|
"loss": 98.41602325439453,
|
|
"loss/core": 98.41602325439453,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6071736812591553,
|
|
"rewards/margins": 0.8191731572151184,
|
|
"rewards/rejected": 0.7880005240440369,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 448.0,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -1.8817628622055054,
|
|
"logits/rejected": -1.8192886114120483,
|
|
"logps/chosen": -0.2736721634864807,
|
|
"logps/rejected": -0.2607767879962921,
|
|
"loss": 101.3716049194336,
|
|
"loss/core": 101.3716049194336,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6075308322906494,
|
|
"rewards/margins": -0.15226136147975922,
|
|
"rewards/rejected": 2.7597920894622803,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 940.0,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -1.764796495437622,
|
|
"logits/rejected": -1.7796127796173096,
|
|
"logps/chosen": -0.28747016191482544,
|
|
"logps/rejected": -0.29904478788375854,
|
|
"loss": 97.57106018066406,
|
|
"loss/core": 97.57106018066406,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.577658176422119,
|
|
"rewards/margins": 1.2913696765899658,
|
|
"rewards/rejected": 1.2862884998321533,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 484.0,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -1.860839605331421,
|
|
"logits/rejected": -1.8101718425750732,
|
|
"logps/chosen": -0.2720187306404114,
|
|
"logps/rejected": -0.26114434003829956,
|
|
"loss": 97.21582794189453,
|
|
"loss/core": 97.21582794189453,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2341485023498535,
|
|
"rewards/margins": 1.7361303567886353,
|
|
"rewards/rejected": 1.4980182647705078,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 128.0,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -1.9565016031265259,
|
|
"logits/rejected": -1.9064795970916748,
|
|
"logps/chosen": -0.28875815868377686,
|
|
"logps/rejected": -0.2922629415988922,
|
|
"loss": 99.18455505371094,
|
|
"loss/core": 99.18455505371094,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.042085886001587,
|
|
"rewards/margins": 0.4169158339500427,
|
|
"rewards/rejected": 0.625170111656189,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -1.9324171543121338,
|
|
"logits/rejected": -1.926792860031128,
|
|
"logps/chosen": -0.28232958912849426,
|
|
"logps/rejected": -0.285550057888031,
|
|
"loss": 97.28160095214844,
|
|
"loss/core": 97.28160095214844,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8508822917938232,
|
|
"rewards/margins": 1.6247138977050781,
|
|
"rewards/rejected": 1.2261682748794556,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.651599645614624,
|
|
"logits/rejected": -1.645272970199585,
|
|
"logps/chosen": -0.2920708656311035,
|
|
"logps/rejected": -0.2902786135673523,
|
|
"loss": 97.38813781738281,
|
|
"loss/core": 97.38813781738281,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6577506065368652,
|
|
"rewards/margins": 1.697007417678833,
|
|
"rewards/rejected": 0.9607435464859009,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 158.0,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.7238547801971436,
|
|
"logits/rejected": -1.7459415197372437,
|
|
"logps/chosen": -0.3239200711250305,
|
|
"logps/rejected": -0.31623491644859314,
|
|
"loss": 97.3945541381836,
|
|
"loss/core": 97.3945541381836,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.610980987548828,
|
|
"rewards/margins": 1.3865725994110107,
|
|
"rewards/rejected": 1.2244083881378174,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 107.5,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -1.8025785684585571,
|
|
"logits/rejected": -1.8798892498016357,
|
|
"logps/chosen": -0.25078368186950684,
|
|
"logps/rejected": -0.270027220249176,
|
|
"loss": 98.32032775878906,
|
|
"loss/core": 98.32032775878906,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.282686948776245,
|
|
"rewards/margins": 1.18528151512146,
|
|
"rewards/rejected": 2.097405195236206,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 274.0,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -1.8228957653045654,
|
|
"logits/rejected": -1.8074302673339844,
|
|
"logps/chosen": -0.31188255548477173,
|
|
"logps/rejected": -0.32124748826026917,
|
|
"loss": 97.54026794433594,
|
|
"loss/core": 97.54026794433594,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8372724056243896,
|
|
"rewards/margins": 1.347383737564087,
|
|
"rewards/rejected": 1.4898887872695923,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 250.0,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -1.8536853790283203,
|
|
"logits/rejected": -1.8146007061004639,
|
|
"logps/chosen": -0.27515512704849243,
|
|
"logps/rejected": -0.2840956151485443,
|
|
"loss": 98.03938293457031,
|
|
"loss/core": 98.03938293457031,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1946370601654053,
|
|
"rewards/margins": 1.4905591011047363,
|
|
"rewards/rejected": 1.7040780782699585,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 276.0,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -1.770841360092163,
|
|
"logits/rejected": -1.7254314422607422,
|
|
"logps/chosen": -0.3026730716228485,
|
|
"logps/rejected": -0.30042824149131775,
|
|
"loss": 98.44063568115234,
|
|
"loss/core": 98.44063568115234,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8839222192764282,
|
|
"rewards/margins": 0.8008639216423035,
|
|
"rewards/rejected": 1.0830583572387695,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -1.822609305381775,
|
|
"logits/rejected": -1.8853318691253662,
|
|
"logps/chosen": -0.29000169038772583,
|
|
"logps/rejected": -0.2747119069099426,
|
|
"loss": 96.0133285522461,
|
|
"loss/core": 96.0133285522461,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.628028154373169,
|
|
"rewards/margins": 2.211141586303711,
|
|
"rewards/rejected": 1.416886568069458,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 1568.0,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -1.956534743309021,
|
|
"logits/rejected": -1.9918060302734375,
|
|
"logps/chosen": -0.2819087505340576,
|
|
"logps/rejected": -0.3088639974594116,
|
|
"loss": 97.88602447509766,
|
|
"loss/core": 97.88602447509766,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8525536060333252,
|
|
"rewards/margins": 1.1093358993530273,
|
|
"rewards/rejected": 0.7432177066802979,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 177.0,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -1.6948636770248413,
|
|
"logits/rejected": -1.711146354675293,
|
|
"logps/chosen": -0.29289984703063965,
|
|
"logps/rejected": -0.28607800602912903,
|
|
"loss": 98.97444152832031,
|
|
"loss/core": 98.97444152832031,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9021542072296143,
|
|
"rewards/margins": 0.5686413049697876,
|
|
"rewards/rejected": 1.3335130214691162,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 169.0,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -1.7635622024536133,
|
|
"logits/rejected": -1.7835123538970947,
|
|
"logps/chosen": -0.29360511898994446,
|
|
"logps/rejected": -0.29403769969940186,
|
|
"loss": 98.54844665527344,
|
|
"loss/core": 98.54844665527344,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7672475576400757,
|
|
"rewards/margins": 0.739784836769104,
|
|
"rewards/rejected": 1.0274627208709717,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 1576.0,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -1.9231055974960327,
|
|
"logits/rejected": -1.9007583856582642,
|
|
"logps/chosen": -0.27654072642326355,
|
|
"logps/rejected": -0.28899580240249634,
|
|
"loss": 96.15101623535156,
|
|
"loss/core": 96.15101623535156,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6142005920410156,
|
|
"rewards/margins": 2.4411978721618652,
|
|
"rewards/rejected": 1.1730026006698608,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.6372286081314087,
|
|
"logits/rejected": -1.6989704370498657,
|
|
"logps/chosen": -0.3004458248615265,
|
|
"logps/rejected": -0.29450005292892456,
|
|
"loss": 97.6203842163086,
|
|
"loss/core": 97.6203842163086,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.332899570465088,
|
|
"rewards/margins": 1.2636841535568237,
|
|
"rewards/rejected": 1.069215178489685,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -1.8383357524871826,
|
|
"logits/rejected": -1.847869634628296,
|
|
"logps/chosen": -0.28603678941726685,
|
|
"logps/rejected": -0.2877523601055145,
|
|
"loss": 98.42637634277344,
|
|
"loss/core": 98.42637634277344,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4403929710388184,
|
|
"rewards/margins": 0.8402155041694641,
|
|
"rewards/rejected": 1.600177526473999,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 68.5,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -1.909732460975647,
|
|
"logits/rejected": -1.9944489002227783,
|
|
"logps/chosen": -0.28810587525367737,
|
|
"logps/rejected": -0.2701200842857361,
|
|
"loss": 98.69931030273438,
|
|
"loss/core": 98.69931030273438,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3960041999816895,
|
|
"rewards/margins": 0.8098465800285339,
|
|
"rewards/rejected": 1.5861575603485107,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 68.0,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -1.796696662902832,
|
|
"logits/rejected": -1.8159329891204834,
|
|
"logps/chosen": -0.2996280789375305,
|
|
"logps/rejected": -0.30264267325401306,
|
|
"loss": 100.24183654785156,
|
|
"loss/core": 100.24183654785156,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.2289425134658813,
|
|
"rewards/margins": 0.06570056825876236,
|
|
"rewards/rejected": 1.163241982460022,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 122.5,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.6407196521759033,
|
|
"logits/rejected": -1.7502638101577759,
|
|
"logps/chosen": -0.29027777910232544,
|
|
"logps/rejected": -0.2889750599861145,
|
|
"loss": 98.38270568847656,
|
|
"loss/core": 98.38270568847656,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2634711265563965,
|
|
"rewards/margins": 1.1196836233139038,
|
|
"rewards/rejected": 2.143787145614624,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 1012.0,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -1.8677374124526978,
|
|
"logits/rejected": -1.8112701177597046,
|
|
"logps/chosen": -0.2966800630092621,
|
|
"logps/rejected": -0.2986254394054413,
|
|
"loss": 97.82476043701172,
|
|
"loss/core": 97.82476043701172,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3230066299438477,
|
|
"rewards/margins": 1.1454260349273682,
|
|
"rewards/rejected": 1.177580714225769,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 115.0,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -1.7707321643829346,
|
|
"logits/rejected": -1.7221782207489014,
|
|
"logps/chosen": -0.28835004568099976,
|
|
"logps/rejected": -0.3050932288169861,
|
|
"loss": 97.6014633178711,
|
|
"loss/core": 97.6014633178711,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0437865257263184,
|
|
"rewards/margins": 1.259574294090271,
|
|
"rewards/rejected": 0.7842122316360474,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 536.0,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -1.8891651630401611,
|
|
"logits/rejected": -1.8469898700714111,
|
|
"logps/chosen": -0.29696494340896606,
|
|
"logps/rejected": -0.29243922233581543,
|
|
"loss": 98.27204132080078,
|
|
"loss/core": 98.27204132080078,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.607239246368408,
|
|
"rewards/margins": 0.9751089215278625,
|
|
"rewards/rejected": 1.6321302652359009,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 2144.0,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -1.8728892803192139,
|
|
"logits/rejected": -1.9022899866104126,
|
|
"logps/chosen": -0.2866348624229431,
|
|
"logps/rejected": -0.28832441568374634,
|
|
"loss": 100.2232437133789,
|
|
"loss/core": 100.2232437133789,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.4807735681533813,
|
|
"rewards/margins": 0.033438198268413544,
|
|
"rewards/rejected": 1.4473353624343872,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 134.0,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.1759440898895264,
|
|
"logits/rejected": -2.1741747856140137,
|
|
"logps/chosen": -0.24452364444732666,
|
|
"logps/rejected": -0.25654464960098267,
|
|
"loss": 98.74320983886719,
|
|
"loss/core": 98.74320983886719,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3100438117980957,
|
|
"rewards/margins": 0.6354137659072876,
|
|
"rewards/rejected": 0.6746301651000977,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 1488.0,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.5932027101516724,
|
|
"logits/rejected": -1.6199067831039429,
|
|
"logps/chosen": -0.3133957087993622,
|
|
"logps/rejected": -0.31795522570610046,
|
|
"loss": 95.31610107421875,
|
|
"loss/core": 95.31610107421875,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.7138900756835938,
|
|
"rewards/margins": 2.6641526222229004,
|
|
"rewards/rejected": 1.0497372150421143,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -1.7853899002075195,
|
|
"logits/rejected": -1.8296200037002563,
|
|
"logps/chosen": -0.2687693238258362,
|
|
"logps/rejected": -0.2597236633300781,
|
|
"loss": 97.58959197998047,
|
|
"loss/core": 97.58959197998047,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1659064292907715,
|
|
"rewards/margins": 1.2582628726959229,
|
|
"rewards/rejected": 0.9076434969902039,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 1936.0,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -1.829178810119629,
|
|
"logits/rejected": -1.731296181678772,
|
|
"logps/chosen": -0.27323228120803833,
|
|
"logps/rejected": -0.2782127559185028,
|
|
"loss": 99.45023345947266,
|
|
"loss/core": 99.45023345947266,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.610325574874878,
|
|
"rewards/margins": 0.5705730319023132,
|
|
"rewards/rejected": 2.03975248336792,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 125.0,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.7197242975234985,
|
|
"logits/rejected": -1.8159154653549194,
|
|
"logps/chosen": -0.26879817247390747,
|
|
"logps/rejected": -0.2760879397392273,
|
|
"loss": 96.94550323486328,
|
|
"loss/core": 96.94550323486328,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9166643619537354,
|
|
"rewards/margins": 1.574958086013794,
|
|
"rewards/rejected": 1.3417065143585205,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 104.5,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -1.7181727886199951,
|
|
"logits/rejected": -1.6899751424789429,
|
|
"logps/chosen": -0.2890922427177429,
|
|
"logps/rejected": -0.2932893633842468,
|
|
"loss": 99.34120178222656,
|
|
"loss/core": 99.34120178222656,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5913293361663818,
|
|
"rewards/margins": 0.36940068006515503,
|
|
"rewards/rejected": 1.221928596496582,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 484.0,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -1.967686653137207,
|
|
"logits/rejected": -1.9203250408172607,
|
|
"logps/chosen": -0.2864150404930115,
|
|
"logps/rejected": -0.28275853395462036,
|
|
"loss": 97.01065826416016,
|
|
"loss/core": 97.01065826416016,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.6977322101593018,
|
|
"rewards/margins": 2.1541683673858643,
|
|
"rewards/rejected": 1.5435636043548584,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 82.5,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -1.899263620376587,
|
|
"logits/rejected": -1.861311674118042,
|
|
"logps/chosen": -0.27477145195007324,
|
|
"logps/rejected": -0.27060219645500183,
|
|
"loss": 98.25178527832031,
|
|
"loss/core": 98.25178527832031,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.069042682647705,
|
|
"rewards/margins": 0.9386270642280579,
|
|
"rewards/rejected": 1.130415678024292,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 1248.0,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -1.807976484298706,
|
|
"logits/rejected": -1.8044418096542358,
|
|
"logps/chosen": -0.30185666680336,
|
|
"logps/rejected": -0.3223247230052948,
|
|
"loss": 97.50824737548828,
|
|
"loss/core": 97.50824737548828,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1243371963500977,
|
|
"rewards/margins": 1.4735743999481201,
|
|
"rewards/rejected": 0.6507625579833984,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 644.0,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -1.8412240743637085,
|
|
"logits/rejected": -1.9032318592071533,
|
|
"logps/chosen": -0.27988195419311523,
|
|
"logps/rejected": -0.28126993775367737,
|
|
"loss": 97.8116455078125,
|
|
"loss/core": 97.8116455078125,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.042039394378662,
|
|
"rewards/margins": 1.1677625179290771,
|
|
"rewards/rejected": 0.8742771148681641,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 202.0,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -1.8139426708221436,
|
|
"logits/rejected": -1.8884384632110596,
|
|
"logps/chosen": -0.28163427114486694,
|
|
"logps/rejected": -0.2937542796134949,
|
|
"loss": 98.46784210205078,
|
|
"loss/core": 98.46784210205078,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6491235494613647,
|
|
"rewards/margins": 0.817004382610321,
|
|
"rewards/rejected": 0.8321191668510437,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 165.0,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -1.8352000713348389,
|
|
"logits/rejected": -1.8296496868133545,
|
|
"logps/chosen": -0.26140180230140686,
|
|
"logps/rejected": -0.2555782198905945,
|
|
"loss": 98.01313018798828,
|
|
"loss/core": 98.01313018798828,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9879388809204102,
|
|
"rewards/margins": 1.031970500946045,
|
|
"rewards/rejected": 0.9559683799743652,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 165.0,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -1.7930519580841064,
|
|
"logits/rejected": -1.874668836593628,
|
|
"logps/chosen": -0.2808493673801422,
|
|
"logps/rejected": -0.29185348749160767,
|
|
"loss": 99.32098388671875,
|
|
"loss/core": 99.32098388671875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8493973016738892,
|
|
"rewards/margins": 0.49081817269325256,
|
|
"rewards/rejected": 1.358579397201538,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 1024.0,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -1.8572410345077515,
|
|
"logits/rejected": -1.9341695308685303,
|
|
"logps/chosen": -0.27888017892837524,
|
|
"logps/rejected": -0.280597448348999,
|
|
"loss": 97.60267639160156,
|
|
"loss/core": 97.60267639160156,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.646106481552124,
|
|
"rewards/margins": 1.358185052871704,
|
|
"rewards/rejected": 1.28792142868042,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 362.0,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -1.8237988948822021,
|
|
"logits/rejected": -1.8687849044799805,
|
|
"logps/chosen": -0.28639885783195496,
|
|
"logps/rejected": -0.2829182744026184,
|
|
"loss": 96.79151916503906,
|
|
"loss/core": 96.79151916503906,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.860142946243286,
|
|
"rewards/margins": 2.0410332679748535,
|
|
"rewards/rejected": 0.8191097378730774,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 324.0,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -1.913863182067871,
|
|
"logits/rejected": -2.0241215229034424,
|
|
"logps/chosen": -0.28453686833381653,
|
|
"logps/rejected": -0.2855282425880432,
|
|
"loss": 98.85942840576172,
|
|
"loss/core": 98.85942840576172,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.647334098815918,
|
|
"rewards/margins": 0.5878774523735046,
|
|
"rewards/rejected": 1.059456706047058,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -1.7168223857879639,
|
|
"logits/rejected": -1.7345936298370361,
|
|
"logps/chosen": -0.2586119472980499,
|
|
"logps/rejected": -0.2558966279029846,
|
|
"loss": 98.92908477783203,
|
|
"loss/core": 98.92908477783203,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0151703357696533,
|
|
"rewards/margins": 0.7727279663085938,
|
|
"rewards/rejected": 2.2424423694610596,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 213.0,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -1.8439680337905884,
|
|
"logits/rejected": -1.8838622570037842,
|
|
"logps/chosen": -0.27940452098846436,
|
|
"logps/rejected": -0.28964605927467346,
|
|
"loss": 99.63495635986328,
|
|
"loss/core": 99.63495635986328,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0571694374084473,
|
|
"rewards/margins": 0.45843571424484253,
|
|
"rewards/rejected": 1.5987335443496704,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 276.0,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -1.8193256855010986,
|
|
"logits/rejected": -1.8221651315689087,
|
|
"logps/chosen": -0.2699766159057617,
|
|
"logps/rejected": -0.2746611535549164,
|
|
"loss": 98.85408782958984,
|
|
"loss/core": 98.85408782958984,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.1804919242858887,
|
|
"rewards/margins": 0.6561237573623657,
|
|
"rewards/rejected": 1.524368405342102,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 87.0,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.6936248540878296,
|
|
"logits/rejected": -1.7653439044952393,
|
|
"logps/chosen": -0.30715465545654297,
|
|
"logps/rejected": -0.30202072858810425,
|
|
"loss": 98.37455749511719,
|
|
"loss/core": 98.37455749511719,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.762188196182251,
|
|
"rewards/margins": 0.8616089820861816,
|
|
"rewards/rejected": 0.9005790948867798,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 241.0,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -1.9225870370864868,
|
|
"logits/rejected": -1.9785583019256592,
|
|
"logps/chosen": -0.2938076853752136,
|
|
"logps/rejected": -0.28969907760620117,
|
|
"loss": 98.27570343017578,
|
|
"loss/core": 98.27570343017578,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 1.694026231765747,
|
|
"rewards/margins": 0.8879431486129761,
|
|
"rewards/rejected": 0.8060829043388367,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 175.0,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -1.6314131021499634,
|
|
"logits/rejected": -1.6901371479034424,
|
|
"logps/chosen": -0.29122158885002136,
|
|
"logps/rejected": -0.2936597466468811,
|
|
"loss": 96.5455093383789,
|
|
"loss/core": 96.5455093383789,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9411308765411377,
|
|
"rewards/margins": 2.1050190925598145,
|
|
"rewards/rejected": 0.8361119031906128,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -1.834407091140747,
|
|
"logits/rejected": -1.8702154159545898,
|
|
"logps/chosen": -0.3132235109806061,
|
|
"logps/rejected": -0.30386728048324585,
|
|
"loss": 98.06851196289062,
|
|
"loss/core": 98.06851196289062,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3328118324279785,
|
|
"rewards/margins": 1.0983474254608154,
|
|
"rewards/rejected": 1.234464406967163,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 174.0,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.085736036300659,
|
|
"logits/rejected": -2.084230899810791,
|
|
"logps/chosen": -0.27615100145339966,
|
|
"logps/rejected": -0.2881914973258972,
|
|
"loss": 98.22357940673828,
|
|
"loss/core": 98.22357940673828,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.096590518951416,
|
|
"rewards/margins": 0.9094375371932983,
|
|
"rewards/rejected": 1.1871533393859863,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 167.0,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -1.6657861471176147,
|
|
"logits/rejected": -1.7485307455062866,
|
|
"logps/chosen": -0.33018797636032104,
|
|
"logps/rejected": -0.3208489716053009,
|
|
"loss": 98.79418182373047,
|
|
"loss/core": 98.79418182373047,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 0.9962074160575867,
|
|
"rewards/margins": 0.6192155480384827,
|
|
"rewards/rejected": 0.3769919276237488,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 358.0,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -2.0174975395202637,
|
|
"logits/rejected": -2.013953447341919,
|
|
"logps/chosen": -0.2835124135017395,
|
|
"logps/rejected": -0.2982017695903778,
|
|
"loss": 97.60393524169922,
|
|
"loss/core": 97.60393524169922,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0437941551208496,
|
|
"rewards/margins": 1.3167150020599365,
|
|
"rewards/rejected": 0.7270792722702026,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 728.0,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -1.7338950634002686,
|
|
"logits/rejected": -1.8055951595306396,
|
|
"logps/chosen": -0.2909316420555115,
|
|
"logps/rejected": -0.2653912901878357,
|
|
"loss": 99.46458435058594,
|
|
"loss/core": 99.46458435058594,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2419204711914062,
|
|
"rewards/margins": 0.5987423658370972,
|
|
"rewards/rejected": 1.6431777477264404,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -1.812638521194458,
|
|
"logits/rejected": -1.8498079776763916,
|
|
"logps/chosen": -0.2791460454463959,
|
|
"logps/rejected": -0.2673080265522003,
|
|
"loss": 95.21307373046875,
|
|
"loss/core": 95.21307373046875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.406863212585449,
|
|
"rewards/margins": 2.995854139328003,
|
|
"rewards/rejected": 1.4110084772109985,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 194.0,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.0415005683898926,
|
|
"logits/rejected": -1.9983999729156494,
|
|
"logps/chosen": -0.2776305079460144,
|
|
"logps/rejected": -0.29190298914909363,
|
|
"loss": 98.83068084716797,
|
|
"loss/core": 98.83068084716797,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6318155527114868,
|
|
"rewards/margins": 0.6132717728614807,
|
|
"rewards/rejected": 1.0185438394546509,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 1056.0,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -1.7515466213226318,
|
|
"logits/rejected": -1.754542589187622,
|
|
"logps/chosen": -0.29860061407089233,
|
|
"logps/rejected": -0.2998600900173187,
|
|
"loss": 97.38858032226562,
|
|
"loss/core": 97.38858032226562,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.053305149078369,
|
|
"rewards/margins": 1.6653683185577393,
|
|
"rewards/rejected": 1.3879367113113403,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 286.0,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.676851511001587,
|
|
"logits/rejected": -1.6223485469818115,
|
|
"logps/chosen": -0.29457855224609375,
|
|
"logps/rejected": -0.29545649886131287,
|
|
"loss": 97.41128540039062,
|
|
"loss/core": 97.41128540039062,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.970858097076416,
|
|
"rewards/margins": 1.3911173343658447,
|
|
"rewards/rejected": 1.57974112033844,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 1464.0,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.6469995975494385,
|
|
"logits/rejected": -1.6655452251434326,
|
|
"logps/chosen": -0.29804956912994385,
|
|
"logps/rejected": -0.2897404432296753,
|
|
"loss": 96.7783432006836,
|
|
"loss/core": 96.7783432006836,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.4604811668395996,
|
|
"rewards/margins": 1.8570493459701538,
|
|
"rewards/rejected": 1.6034319400787354,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 474.0,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.6406848430633545,
|
|
"logits/rejected": -1.831834077835083,
|
|
"logps/chosen": -0.3217054009437561,
|
|
"logps/rejected": -0.31396520137786865,
|
|
"loss": 96.65595245361328,
|
|
"loss/core": 96.65595245361328,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.475750684738159,
|
|
"rewards/margins": 1.9972509145736694,
|
|
"rewards/rejected": 1.4784997701644897,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 306.0,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -1.6828476190567017,
|
|
"logits/rejected": -1.7117191553115845,
|
|
"logps/chosen": -0.2919991612434387,
|
|
"logps/rejected": -0.3055281937122345,
|
|
"loss": 96.70384216308594,
|
|
"loss/core": 96.70384216308594,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.7047760486602783,
|
|
"rewards/margins": 1.8250391483306885,
|
|
"rewards/rejected": 1.879737138748169,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 127.0,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.5566418170928955,
|
|
"logits/rejected": -1.6399599313735962,
|
|
"logps/chosen": -0.2965025007724762,
|
|
"logps/rejected": -0.2912459373474121,
|
|
"loss": 97.3532943725586,
|
|
"loss/core": 97.3532943725586,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.3075928688049316,
|
|
"rewards/margins": 1.3737890720367432,
|
|
"rewards/rejected": 0.9338043332099915,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 1104.0,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -1.879783272743225,
|
|
"logits/rejected": -1.9588508605957031,
|
|
"logps/chosen": -0.2798837423324585,
|
|
"logps/rejected": -0.2600264847278595,
|
|
"loss": 96.83983612060547,
|
|
"loss/core": 96.83983612060547,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.0017495155334473,
|
|
"rewards/margins": 1.8324419260025024,
|
|
"rewards/rejected": 1.1693079471588135,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 592.0,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -1.8083826303482056,
|
|
"logits/rejected": -1.7853968143463135,
|
|
"logps/chosen": -0.2943006753921509,
|
|
"logps/rejected": -0.3146829903125763,
|
|
"loss": 99.76537322998047,
|
|
"loss/core": 99.76537322998047,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8425248861312866,
|
|
"rewards/margins": 0.3144264817237854,
|
|
"rewards/rejected": 1.5280983448028564,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 150.0,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -1.9771804809570312,
|
|
"logits/rejected": -2.06240177154541,
|
|
"logps/chosen": -0.2760615050792694,
|
|
"logps/rejected": -0.271167129278183,
|
|
"loss": 97.69608306884766,
|
|
"loss/core": 97.69608306884766,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3518784046173096,
|
|
"rewards/margins": 1.3141744136810303,
|
|
"rewards/rejected": 1.0377036333084106,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.0386388301849365,
|
|
"logits/rejected": -2.0184037685394287,
|
|
"logps/chosen": -0.2914060056209564,
|
|
"logps/rejected": -0.2953184247016907,
|
|
"loss": 98.33658599853516,
|
|
"loss/core": 98.33658599853516,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7446434497833252,
|
|
"rewards/margins": 0.8531507253646851,
|
|
"rewards/rejected": 0.8914925456047058,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 243.0,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -1.876051902770996,
|
|
"logits/rejected": -1.899723768234253,
|
|
"logps/chosen": -0.30638667941093445,
|
|
"logps/rejected": -0.2942095100879669,
|
|
"loss": 98.0177993774414,
|
|
"loss/core": 98.0177993774414,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0851686000823975,
|
|
"rewards/margins": 1.030152678489685,
|
|
"rewards/rejected": 1.0550158023834229,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 274.0,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -1.882607102394104,
|
|
"logits/rejected": -1.9915786981582642,
|
|
"logps/chosen": -0.2932814061641693,
|
|
"logps/rejected": -0.2904340624809265,
|
|
"loss": 96.98285675048828,
|
|
"loss/core": 96.98285675048828,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6567623615264893,
|
|
"rewards/margins": 1.7608906030654907,
|
|
"rewards/rejected": 0.8958717584609985,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 414.0,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.7806377410888672,
|
|
"logits/rejected": -1.7007198333740234,
|
|
"logps/chosen": -0.2761852443218231,
|
|
"logps/rejected": -0.30116838216781616,
|
|
"loss": 97.001220703125,
|
|
"loss/core": 97.001220703125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.212673664093018,
|
|
"rewards/margins": 2.1719820499420166,
|
|
"rewards/rejected": 2.04069185256958,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 592.0,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -1.5999929904937744,
|
|
"logits/rejected": -1.6960639953613281,
|
|
"logps/chosen": -0.2828037738800049,
|
|
"logps/rejected": -0.2773839831352234,
|
|
"loss": 97.8866195678711,
|
|
"loss/core": 97.8866195678711,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6385066509246826,
|
|
"rewards/margins": 1.7813966274261475,
|
|
"rewards/rejected": 1.857109785079956,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 48.0,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -1.815176010131836,
|
|
"logits/rejected": -1.9088054895401,
|
|
"logps/chosen": -0.31443700194358826,
|
|
"logps/rejected": -0.29087454080581665,
|
|
"loss": 97.86416625976562,
|
|
"loss/core": 97.86416625976562,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0255415439605713,
|
|
"rewards/margins": 1.2207753658294678,
|
|
"rewards/rejected": 0.8047658205032349,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -1.6068187952041626,
|
|
"logits/rejected": -1.6211055517196655,
|
|
"logps/chosen": -0.2836398482322693,
|
|
"logps/rejected": -0.3053371012210846,
|
|
"loss": 98.37687683105469,
|
|
"loss/core": 98.37687683105469,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.53507399559021,
|
|
"rewards/margins": 0.8702079057693481,
|
|
"rewards/rejected": 1.6648662090301514,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 310.0,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -1.8465595245361328,
|
|
"logits/rejected": -1.9376262426376343,
|
|
"logps/chosen": -0.3172188401222229,
|
|
"logps/rejected": -0.3324028551578522,
|
|
"loss": 96.29479217529297,
|
|
"loss/core": 96.29479217529297,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1222901344299316,
|
|
"rewards/margins": 2.14201021194458,
|
|
"rewards/rejected": 0.9802799224853516,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 980.0,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -1.9088995456695557,
|
|
"logits/rejected": -1.8524242639541626,
|
|
"logps/chosen": -0.28945475816726685,
|
|
"logps/rejected": -0.29769039154052734,
|
|
"loss": 95.80438995361328,
|
|
"loss/core": 95.80438995361328,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.41683292388916,
|
|
"rewards/margins": 2.928234338760376,
|
|
"rewards/rejected": 1.4885984659194946,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 54.75,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -1.7017314434051514,
|
|
"logits/rejected": -1.697610855102539,
|
|
"logps/chosen": -0.28460589051246643,
|
|
"logps/rejected": -0.28730088472366333,
|
|
"loss": 97.26568603515625,
|
|
"loss/core": 97.26568603515625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.064678907394409,
|
|
"rewards/margins": 1.5623985528945923,
|
|
"rewards/rejected": 1.5022804737091064,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 262.0,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -1.7895727157592773,
|
|
"logits/rejected": -1.7456676959991455,
|
|
"logps/chosen": -0.28233015537261963,
|
|
"logps/rejected": -0.27622172236442566,
|
|
"loss": 96.65437316894531,
|
|
"loss/core": 96.65437316894531,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.126199722290039,
|
|
"rewards/margins": 2.0186972618103027,
|
|
"rewards/rejected": 1.1075026988983154,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 83.5,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -1.7324750423431396,
|
|
"logits/rejected": -1.6515331268310547,
|
|
"logps/chosen": -0.2788090407848358,
|
|
"logps/rejected": -0.28251534700393677,
|
|
"loss": 100.15641784667969,
|
|
"loss/core": 100.15641784667969,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.587709665298462,
|
|
"rewards/margins": 0.028971601277589798,
|
|
"rewards/rejected": 1.5587379932403564,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 612.0,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.0007410049438477,
|
|
"logits/rejected": -1.9063104391098022,
|
|
"logps/chosen": -0.2854381501674652,
|
|
"logps/rejected": -0.28918930888175964,
|
|
"loss": 97.22596740722656,
|
|
"loss/core": 97.22596740722656,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.619149923324585,
|
|
"rewards/margins": 1.7024046182632446,
|
|
"rewards/rejected": 0.9167453646659851,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 203.0,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.5841925144195557,
|
|
"logits/rejected": -1.5888211727142334,
|
|
"logps/chosen": -0.276597261428833,
|
|
"logps/rejected": -0.29771602153778076,
|
|
"loss": 97.82579803466797,
|
|
"loss/core": 97.82579803466797,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.763697862625122,
|
|
"rewards/margins": 1.1396576166152954,
|
|
"rewards/rejected": 1.6240402460098267,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 112.0,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -1.8474057912826538,
|
|
"logits/rejected": -1.8652210235595703,
|
|
"logps/chosen": -0.3112329840660095,
|
|
"logps/rejected": -0.2748587727546692,
|
|
"loss": 95.9908447265625,
|
|
"loss/core": 95.9908447265625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2994256019592285,
|
|
"rewards/margins": 2.3030340671539307,
|
|
"rewards/rejected": 0.9963914155960083,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 300.0,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -1.7741531133651733,
|
|
"logits/rejected": -1.8085333108901978,
|
|
"logps/chosen": -0.27475112676620483,
|
|
"logps/rejected": -0.27942371368408203,
|
|
"loss": 97.57888793945312,
|
|
"loss/core": 97.57888793945312,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8250153064727783,
|
|
"rewards/margins": 1.2808738946914673,
|
|
"rewards/rejected": 1.5441416501998901,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 302.0,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -1.8415495157241821,
|
|
"logits/rejected": -1.8162124156951904,
|
|
"logps/chosen": -0.287056028842926,
|
|
"logps/rejected": -0.30101385712623596,
|
|
"loss": 98.02638244628906,
|
|
"loss/core": 98.02638244628906,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.785414695739746,
|
|
"rewards/margins": 1.0829885005950928,
|
|
"rewards/rejected": 1.702426552772522,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 752.0,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -1.7960054874420166,
|
|
"logits/rejected": -1.8172178268432617,
|
|
"logps/chosen": -0.28870075941085815,
|
|
"logps/rejected": -0.2848988175392151,
|
|
"loss": 97.02275085449219,
|
|
"loss/core": 97.02275085449219,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.343677043914795,
|
|
"rewards/margins": 1.6194202899932861,
|
|
"rewards/rejected": 1.7242568731307983,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 87.5,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -1.8886982202529907,
|
|
"logits/rejected": -1.9144996404647827,
|
|
"logps/chosen": -0.30019474029541016,
|
|
"logps/rejected": -0.3224802017211914,
|
|
"loss": 97.74614715576172,
|
|
"loss/core": 97.74614715576172,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.847407579421997,
|
|
"rewards/margins": 1.279578685760498,
|
|
"rewards/rejected": 0.5678285360336304,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 59.75,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -1.974991798400879,
|
|
"logits/rejected": -2.0810978412628174,
|
|
"logps/chosen": -0.26656287908554077,
|
|
"logps/rejected": -0.27996402978897095,
|
|
"loss": 99.16670989990234,
|
|
"loss/core": 99.16670989990234,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4196202754974365,
|
|
"rewards/margins": 0.45080775022506714,
|
|
"rewards/rejected": 0.9688124656677246,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -1.9742958545684814,
|
|
"logits/rejected": -1.868665337562561,
|
|
"logps/chosen": -0.2701478898525238,
|
|
"logps/rejected": -0.28200751543045044,
|
|
"loss": 98.4051284790039,
|
|
"loss/core": 98.4051284790039,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.223400115966797,
|
|
"rewards/margins": 0.8472920656204224,
|
|
"rewards/rejected": 1.3761080503463745,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 102.0,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -1.8315715789794922,
|
|
"logits/rejected": -1.8810384273529053,
|
|
"logps/chosen": -0.2877020239830017,
|
|
"logps/rejected": -0.28643354773521423,
|
|
"loss": 95.52645111083984,
|
|
"loss/core": 95.52645111083984,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6193325519561768,
|
|
"rewards/margins": 2.5816495418548584,
|
|
"rewards/rejected": 1.0376828908920288,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -1.7726303339004517,
|
|
"logits/rejected": -1.782941222190857,
|
|
"logps/chosen": -0.27482694387435913,
|
|
"logps/rejected": -0.2720108926296234,
|
|
"loss": 98.37557220458984,
|
|
"loss/core": 98.37557220458984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.714624047279358,
|
|
"rewards/margins": 0.8266318440437317,
|
|
"rewards/rejected": 0.8879922032356262,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 230.0,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -1.8123525381088257,
|
|
"logits/rejected": -1.7736984491348267,
|
|
"logps/chosen": -0.2656711935997009,
|
|
"logps/rejected": -0.281463086605072,
|
|
"loss": 98.5457763671875,
|
|
"loss/core": 98.5457763671875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8796374797821045,
|
|
"rewards/margins": 0.7882030606269836,
|
|
"rewards/rejected": 1.0914344787597656,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 408.0,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -1.9366077184677124,
|
|
"logits/rejected": -2.003767728805542,
|
|
"logps/chosen": -0.2733065187931061,
|
|
"logps/rejected": -0.2762848734855652,
|
|
"loss": 95.27114868164062,
|
|
"loss/core": 95.27114868164062,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.252087593078613,
|
|
"rewards/margins": 3.052070140838623,
|
|
"rewards/rejected": 1.2000172138214111,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 366.0,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -1.6373779773712158,
|
|
"logits/rejected": -1.6163759231567383,
|
|
"logps/chosen": -0.2743273377418518,
|
|
"logps/rejected": -0.2724093794822693,
|
|
"loss": 96.03132629394531,
|
|
"loss/core": 96.03132629394531,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.291282653808594,
|
|
"rewards/margins": 2.326350212097168,
|
|
"rewards/rejected": 1.9649326801300049,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.4851322174072266,
|
|
"logits/rejected": -1.566975712776184,
|
|
"logps/chosen": -0.31360191106796265,
|
|
"logps/rejected": -0.30080071091651917,
|
|
"loss": 97.76426696777344,
|
|
"loss/core": 97.76426696777344,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.744011163711548,
|
|
"rewards/margins": 1.2261463403701782,
|
|
"rewards/rejected": 1.5178649425506592,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 124.5,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -1.8698352575302124,
|
|
"logits/rejected": -1.901095986366272,
|
|
"logps/chosen": -0.2850908637046814,
|
|
"logps/rejected": -0.2894435524940491,
|
|
"loss": 99.82987976074219,
|
|
"loss/core": 99.82987976074219,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6998624801635742,
|
|
"rewards/margins": 0.2176227569580078,
|
|
"rewards/rejected": 1.4822396039962769,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 1840.0,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.7007501125335693,
|
|
"logits/rejected": -1.6904194355010986,
|
|
"logps/chosen": -0.3048066198825836,
|
|
"logps/rejected": -0.33391109108924866,
|
|
"loss": 96.62384033203125,
|
|
"loss/core": 96.62384033203125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.886601686477661,
|
|
"rewards/margins": 2.3527793884277344,
|
|
"rewards/rejected": 1.5338222980499268,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.7266985177993774,
|
|
"logits/rejected": -1.8024879693984985,
|
|
"logps/chosen": -0.27022111415863037,
|
|
"logps/rejected": -0.2823617458343506,
|
|
"loss": 96.9268798828125,
|
|
"loss/core": 96.9268798828125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9654762744903564,
|
|
"rewards/margins": 1.7113138437271118,
|
|
"rewards/rejected": 1.2541625499725342,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 636.0,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -1.757341742515564,
|
|
"logits/rejected": -1.8153082132339478,
|
|
"logps/chosen": -0.3024021089076996,
|
|
"logps/rejected": -0.29852160811424255,
|
|
"loss": 97.65138244628906,
|
|
"loss/core": 97.65138244628906,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.330906391143799,
|
|
"rewards/margins": 1.3007912635803223,
|
|
"rewards/rejected": 1.030115008354187,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 170.0,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -1.9056514501571655,
|
|
"logits/rejected": -1.9911472797393799,
|
|
"logps/chosen": -0.29884806275367737,
|
|
"logps/rejected": -0.2987546920776367,
|
|
"loss": 97.85620880126953,
|
|
"loss/core": 97.85620880126953,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9839823246002197,
|
|
"rewards/margins": 1.1865044832229614,
|
|
"rewards/rejected": 0.7974779009819031,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 235.0,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -1.9996535778045654,
|
|
"logits/rejected": -2.021444797515869,
|
|
"logps/chosen": -0.28937220573425293,
|
|
"logps/rejected": -0.29186397790908813,
|
|
"loss": 99.2889404296875,
|
|
"loss/core": 99.2889404296875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6865224838256836,
|
|
"rewards/margins": 0.42166757583618164,
|
|
"rewards/rejected": 1.264854907989502,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 137.0,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -1.9977695941925049,
|
|
"logits/rejected": -1.9729766845703125,
|
|
"logps/chosen": -0.2890729010105133,
|
|
"logps/rejected": -0.2976759374141693,
|
|
"loss": 99.13239288330078,
|
|
"loss/core": 99.13239288330078,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.3376524448394775,
|
|
"rewards/margins": 0.48343315720558167,
|
|
"rewards/rejected": 0.8542193174362183,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 584.0,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -1.7728780508041382,
|
|
"logits/rejected": -1.8141391277313232,
|
|
"logps/chosen": -0.2818259596824646,
|
|
"logps/rejected": -0.28773942589759827,
|
|
"loss": 98.06580352783203,
|
|
"loss/core": 98.06580352783203,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.008873701095581,
|
|
"rewards/margins": 1.0006710290908813,
|
|
"rewards/rejected": 1.0082027912139893,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 93.0,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -1.9082142114639282,
|
|
"logits/rejected": -1.9239473342895508,
|
|
"logps/chosen": -0.2919521927833557,
|
|
"logps/rejected": -0.30435115098953247,
|
|
"loss": 97.76885223388672,
|
|
"loss/core": 97.76885223388672,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.796613931655884,
|
|
"rewards/margins": 1.4722524881362915,
|
|
"rewards/rejected": 1.3243615627288818,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 350.0,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -1.9143612384796143,
|
|
"logits/rejected": -1.8848596811294556,
|
|
"logps/chosen": -0.2826116383075714,
|
|
"logps/rejected": -0.27971625328063965,
|
|
"loss": 98.54499816894531,
|
|
"loss/core": 98.54499816894531,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.364816427230835,
|
|
"rewards/margins": 0.8677363395690918,
|
|
"rewards/rejected": 1.4970804452896118,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 88.0,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.7525949478149414,
|
|
"logits/rejected": -1.8127027750015259,
|
|
"logps/chosen": -0.2727015018463135,
|
|
"logps/rejected": -0.31174588203430176,
|
|
"loss": 95.95095825195312,
|
|
"loss/core": 95.95095825195312,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6611416339874268,
|
|
"rewards/margins": 2.319688320159912,
|
|
"rewards/rejected": 1.3414534330368042,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 170.0,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -1.6614296436309814,
|
|
"logits/rejected": -1.6599369049072266,
|
|
"logps/chosen": -0.2885296940803528,
|
|
"logps/rejected": -0.28654050827026367,
|
|
"loss": 98.60887145996094,
|
|
"loss/core": 98.60887145996094,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6211833953857422,
|
|
"rewards/margins": 0.7235764265060425,
|
|
"rewards/rejected": 0.897607147693634,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 87.0,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -1.8365249633789062,
|
|
"logits/rejected": -1.7715469598770142,
|
|
"logps/chosen": -0.2822161316871643,
|
|
"logps/rejected": -0.29134494066238403,
|
|
"loss": 98.63749694824219,
|
|
"loss/core": 98.63749694824219,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1443328857421875,
|
|
"rewards/margins": 0.7564715147018433,
|
|
"rewards/rejected": 1.3878613710403442,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 116.0,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -1.6401258707046509,
|
|
"logits/rejected": -1.6516834497451782,
|
|
"logps/chosen": -0.2528996169567108,
|
|
"logps/rejected": -0.2761356830596924,
|
|
"loss": 97.95409393310547,
|
|
"loss/core": 97.95409393310547,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8932156562805176,
|
|
"rewards/margins": 1.3494775295257568,
|
|
"rewards/rejected": 1.5437384843826294,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 104.5,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.5243980884552002,
|
|
"logits/rejected": -1.5546753406524658,
|
|
"logps/chosen": -0.3075758218765259,
|
|
"logps/rejected": -0.31810295581817627,
|
|
"loss": 98.9781494140625,
|
|
"loss/core": 98.9781494140625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7742786407470703,
|
|
"rewards/margins": 0.5831277966499329,
|
|
"rewards/rejected": 1.1911507844924927,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -1.8852567672729492,
|
|
"logits/rejected": -1.9301567077636719,
|
|
"logps/chosen": -0.30527764558792114,
|
|
"logps/rejected": -0.292079359292984,
|
|
"loss": 97.71669006347656,
|
|
"loss/core": 97.71669006347656,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9805892705917358,
|
|
"rewards/margins": 1.289915680885315,
|
|
"rewards/rejected": 0.6906735301017761,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 2576.0,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -1.7994495630264282,
|
|
"logits/rejected": -1.8756701946258545,
|
|
"logps/chosen": -0.29662424325942993,
|
|
"logps/rejected": -0.3126930594444275,
|
|
"loss": 99.03868103027344,
|
|
"loss/core": 99.03868103027344,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2323403358459473,
|
|
"rewards/margins": 0.5778122544288635,
|
|
"rewards/rejected": 1.6545279026031494,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 354.0,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -1.8467237949371338,
|
|
"logits/rejected": -1.8279880285263062,
|
|
"logps/chosen": -0.2809194028377533,
|
|
"logps/rejected": -0.2668314576148987,
|
|
"loss": 96.70755004882812,
|
|
"loss/core": 96.70755004882812,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.3989696502685547,
|
|
"rewards/margins": 1.8612855672836304,
|
|
"rewards/rejected": 1.5376842021942139,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 1328.0,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.628369688987732,
|
|
"logits/rejected": -1.6482683420181274,
|
|
"logps/chosen": -0.27590495347976685,
|
|
"logps/rejected": -0.29148903489112854,
|
|
"loss": 96.99464416503906,
|
|
"loss/core": 96.99464416503906,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6098170280456543,
|
|
"rewards/margins": 1.649452805519104,
|
|
"rewards/rejected": 1.9603639841079712,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 122.0,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -1.8616364002227783,
|
|
"logits/rejected": -1.8149951696395874,
|
|
"logps/chosen": -0.28448861837387085,
|
|
"logps/rejected": -0.3247087597846985,
|
|
"loss": 98.08930969238281,
|
|
"loss/core": 98.08930969238281,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8618444204330444,
|
|
"rewards/margins": 1.042851209640503,
|
|
"rewards/rejected": 0.818993091583252,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 312.0,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -1.7652199268341064,
|
|
"logits/rejected": -1.791394829750061,
|
|
"logps/chosen": -0.29653897881507874,
|
|
"logps/rejected": -0.2996441125869751,
|
|
"loss": 99.06591796875,
|
|
"loss/core": 99.06591796875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.1000869274139404,
|
|
"rewards/margins": 0.5941171646118164,
|
|
"rewards/rejected": 1.505969762802124,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 138.0,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -1.9410686492919922,
|
|
"logits/rejected": -1.9667949676513672,
|
|
"logps/chosen": -0.2847985327243805,
|
|
"logps/rejected": -0.2775443196296692,
|
|
"loss": 98.34650421142578,
|
|
"loss/core": 98.34650421142578,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5937460660934448,
|
|
"rewards/margins": 0.865003228187561,
|
|
"rewards/rejected": 0.7287428379058838,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 1392.0,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -1.902320146560669,
|
|
"logits/rejected": -1.948782205581665,
|
|
"logps/chosen": -0.28654351830482483,
|
|
"logps/rejected": -0.29139798879623413,
|
|
"loss": 99.9100570678711,
|
|
"loss/core": 99.9100570678711,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.0183780193328857,
|
|
"rewards/margins": 0.11728978157043457,
|
|
"rewards/rejected": 0.9010881185531616,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 278.0,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -1.8456531763076782,
|
|
"logits/rejected": -1.8964226245880127,
|
|
"logps/chosen": -0.3196776807308197,
|
|
"logps/rejected": -0.31731048226356506,
|
|
"loss": 98.53561401367188,
|
|
"loss/core": 98.53561401367188,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4210125207901,
|
|
"rewards/margins": 0.7529805898666382,
|
|
"rewards/rejected": 0.6680319309234619,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 145.0,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -1.7140204906463623,
|
|
"logits/rejected": -1.7427120208740234,
|
|
"logps/chosen": -0.3081023097038269,
|
|
"logps/rejected": -0.3138575553894043,
|
|
"loss": 99.85032653808594,
|
|
"loss/core": 99.85032653808594,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6136274337768555,
|
|
"rewards/margins": 0.7614258527755737,
|
|
"rewards/rejected": 1.8522018194198608,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 564.0,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -1.7579317092895508,
|
|
"logits/rejected": -1.8332077264785767,
|
|
"logps/chosen": -0.2885438799858093,
|
|
"logps/rejected": -0.2813180088996887,
|
|
"loss": 97.10404968261719,
|
|
"loss/core": 97.10404968261719,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1403250694274902,
|
|
"rewards/margins": 1.6182429790496826,
|
|
"rewards/rejected": 1.5220820903778076,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 668.0,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -1.9111521244049072,
|
|
"logits/rejected": -1.9151312112808228,
|
|
"logps/chosen": -0.3117213845252991,
|
|
"logps/rejected": -0.30328887701034546,
|
|
"loss": 100.52747344970703,
|
|
"loss/core": 100.52747344970703,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.2058069705963135,
|
|
"rewards/margins": -0.05395696312189102,
|
|
"rewards/rejected": 1.2597639560699463,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 286.0,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -1.7503387928009033,
|
|
"logits/rejected": -1.6461431980133057,
|
|
"logps/chosen": -0.30825966596603394,
|
|
"logps/rejected": -0.2929074168205261,
|
|
"loss": 98.85442352294922,
|
|
"loss/core": 98.85442352294922,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.3893883228302,
|
|
"rewards/margins": 0.6656953692436218,
|
|
"rewards/rejected": 1.7236926555633545,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -1.7998508214950562,
|
|
"logits/rejected": -1.8412847518920898,
|
|
"logps/chosen": -0.30292657017707825,
|
|
"logps/rejected": -0.29351457953453064,
|
|
"loss": 97.87567901611328,
|
|
"loss/core": 97.87567901611328,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9885555505752563,
|
|
"rewards/margins": 1.1239904165267944,
|
|
"rewards/rejected": 0.8645650148391724,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 241.0,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -1.7207539081573486,
|
|
"logits/rejected": -1.7541128396987915,
|
|
"logps/chosen": -0.2763254940509796,
|
|
"logps/rejected": -0.27128952741622925,
|
|
"loss": 98.617919921875,
|
|
"loss/core": 98.617919921875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7916427850723267,
|
|
"rewards/margins": 0.7221640944480896,
|
|
"rewards/rejected": 1.0694787502288818,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 430.0,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.6970971822738647,
|
|
"logits/rejected": -1.7250639200210571,
|
|
"logps/chosen": -0.3074544072151184,
|
|
"logps/rejected": -0.280496746301651,
|
|
"loss": 98.15419006347656,
|
|
"loss/core": 98.15419006347656,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7565841674804688,
|
|
"rewards/margins": 1.3622013330459595,
|
|
"rewards/rejected": 2.394383192062378,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 63.0,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -1.8713486194610596,
|
|
"logits/rejected": -1.898063063621521,
|
|
"logps/chosen": -0.3057042062282562,
|
|
"logps/rejected": -0.2998577952384949,
|
|
"loss": 98.88420104980469,
|
|
"loss/core": 98.88420104980469,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.531863808631897,
|
|
"rewards/margins": 0.5855029225349426,
|
|
"rewards/rejected": 0.9463608860969543,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 229.0,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -1.7387259006500244,
|
|
"logits/rejected": -1.7721675634384155,
|
|
"logps/chosen": -0.3017333447933197,
|
|
"logps/rejected": -0.2997443377971649,
|
|
"loss": 98.89266204833984,
|
|
"loss/core": 98.89266204833984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8950691223144531,
|
|
"rewards/margins": 0.5870526432991028,
|
|
"rewards/rejected": 1.3080164194107056,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 84.5,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.5948407649993896,
|
|
"logits/rejected": -1.566903829574585,
|
|
"logps/chosen": -0.30947035551071167,
|
|
"logps/rejected": -0.2902279198169708,
|
|
"loss": 96.91048431396484,
|
|
"loss/core": 96.91048431396484,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.4462432861328125,
|
|
"rewards/margins": 2.065670967102051,
|
|
"rewards/rejected": 1.380571961402893,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 2024.0,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -1.8185821771621704,
|
|
"logits/rejected": -1.7109493017196655,
|
|
"logps/chosen": -0.2856631278991699,
|
|
"logps/rejected": -0.31153404712677,
|
|
"loss": 99.07926940917969,
|
|
"loss/core": 99.07926940917969,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6235830783843994,
|
|
"rewards/margins": 0.6263852715492249,
|
|
"rewards/rejected": 1.9971978664398193,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 300.0,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.777958869934082,
|
|
"logits/rejected": -1.7994849681854248,
|
|
"logps/chosen": -0.30515098571777344,
|
|
"logps/rejected": -0.29699045419692993,
|
|
"loss": 97.77039337158203,
|
|
"loss/core": 97.77039337158203,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.403167486190796,
|
|
"rewards/margins": 1.225862979888916,
|
|
"rewards/rejected": 1.1773045063018799,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 120.5,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -1.8459327220916748,
|
|
"logits/rejected": -1.8227803707122803,
|
|
"logps/chosen": -0.28920984268188477,
|
|
"logps/rejected": -0.2841506004333496,
|
|
"loss": 97.98655700683594,
|
|
"loss/core": 97.98655700683594,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2972395420074463,
|
|
"rewards/margins": 1.0846086740493774,
|
|
"rewards/rejected": 1.2126309871673584,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -1.7277157306671143,
|
|
"logits/rejected": -1.7201650142669678,
|
|
"logps/chosen": -0.27354103326797485,
|
|
"logps/rejected": -0.2667585015296936,
|
|
"loss": 99.72236633300781,
|
|
"loss/core": 99.72236633300781,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2702925205230713,
|
|
"rewards/margins": 0.24597541987895966,
|
|
"rewards/rejected": 2.0243170261383057,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 118.0,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -1.8017603158950806,
|
|
"logits/rejected": -1.7072747945785522,
|
|
"logps/chosen": -0.3080991208553314,
|
|
"logps/rejected": -0.31515592336654663,
|
|
"loss": 100.06008911132812,
|
|
"loss/core": 100.06008911132812,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5322774648666382,
|
|
"rewards/margins": 0.039038121700286865,
|
|
"rewards/rejected": 1.493239402770996,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 302.0,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -1.8767938613891602,
|
|
"logits/rejected": -1.9253695011138916,
|
|
"logps/chosen": -0.2829209566116333,
|
|
"logps/rejected": -0.2839273512363434,
|
|
"loss": 97.97332000732422,
|
|
"loss/core": 97.97332000732422,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1320853233337402,
|
|
"rewards/margins": 1.0345160961151123,
|
|
"rewards/rejected": 1.097569227218628,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 91.0,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -1.7318302392959595,
|
|
"logits/rejected": -1.735965371131897,
|
|
"logps/chosen": -0.28805407881736755,
|
|
"logps/rejected": -0.31744417548179626,
|
|
"loss": 97.60137176513672,
|
|
"loss/core": 97.60137176513672,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9624719619750977,
|
|
"rewards/margins": 1.2788426876068115,
|
|
"rewards/rejected": 0.6836291551589966,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 394.0,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -1.7264869213104248,
|
|
"logits/rejected": -1.8847726583480835,
|
|
"logps/chosen": -0.29893332719802856,
|
|
"logps/rejected": -0.284157931804657,
|
|
"loss": 97.54736328125,
|
|
"loss/core": 97.54736328125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4949123859405518,
|
|
"rewards/margins": 1.2742880582809448,
|
|
"rewards/rejected": 1.220624327659607,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 184.0,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -1.914811134338379,
|
|
"logits/rejected": -1.8754631280899048,
|
|
"logps/chosen": -0.27880120277404785,
|
|
"logps/rejected": -0.30228403210639954,
|
|
"loss": 100.15513610839844,
|
|
"loss/core": 100.15513610839844,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7236683368682861,
|
|
"rewards/margins": 0.013875794596970081,
|
|
"rewards/rejected": 1.7097924947738647,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 596.0,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -1.7310373783111572,
|
|
"logits/rejected": -1.749680757522583,
|
|
"logps/chosen": -0.25445908308029175,
|
|
"logps/rejected": -0.3001643717288971,
|
|
"loss": 97.2359390258789,
|
|
"loss/core": 97.2359390258789,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.406795024871826,
|
|
"rewards/margins": 1.566819190979004,
|
|
"rewards/rejected": 0.8399758338928223,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 1280.0,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -1.8255088329315186,
|
|
"logits/rejected": -1.8728702068328857,
|
|
"logps/chosen": -0.322468101978302,
|
|
"logps/rejected": -0.3011077642440796,
|
|
"loss": 97.09306335449219,
|
|
"loss/core": 97.09306335449219,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.803194284439087,
|
|
"rewards/margins": 1.7630542516708374,
|
|
"rewards/rejected": 1.0401400327682495,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 117.0,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -1.7962318658828735,
|
|
"logits/rejected": -1.8419160842895508,
|
|
"logps/chosen": -0.2930757701396942,
|
|
"logps/rejected": -0.31055259704589844,
|
|
"loss": 98.22071838378906,
|
|
"loss/core": 98.22071838378906,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.4233886003494263,
|
|
"rewards/margins": 0.9614946246147156,
|
|
"rewards/rejected": 0.4618939459323883,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 628.0,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -1.9622621536254883,
|
|
"logits/rejected": -2.055959701538086,
|
|
"logps/chosen": -0.2516617178916931,
|
|
"logps/rejected": -0.24817457795143127,
|
|
"loss": 98.69639587402344,
|
|
"loss/core": 98.69639587402344,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.561449408531189,
|
|
"rewards/margins": 0.6674880385398865,
|
|
"rewards/rejected": 0.8939614295959473,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 330.0,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -1.766444444656372,
|
|
"logits/rejected": -1.7449138164520264,
|
|
"logps/chosen": -0.3007269501686096,
|
|
"logps/rejected": -0.2932063043117523,
|
|
"loss": 97.42681884765625,
|
|
"loss/core": 97.42681884765625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6898107528686523,
|
|
"rewards/margins": 1.440416932106018,
|
|
"rewards/rejected": 1.2493940591812134,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 75.5,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.6410402059555054,
|
|
"logits/rejected": -1.6836130619049072,
|
|
"logps/chosen": -0.29940205812454224,
|
|
"logps/rejected": -0.29790908098220825,
|
|
"loss": 96.29589080810547,
|
|
"loss/core": 96.29589080810547,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.08776330947876,
|
|
"rewards/margins": 2.1483874320983887,
|
|
"rewards/rejected": 1.9393761157989502,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 196.0,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.6578218936920166,
|
|
"logits/rejected": -1.688619613647461,
|
|
"logps/chosen": -0.2899990677833557,
|
|
"logps/rejected": -0.29334208369255066,
|
|
"loss": 98.53080749511719,
|
|
"loss/core": 98.53080749511719,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.461031436920166,
|
|
"rewards/margins": 0.8699079751968384,
|
|
"rewards/rejected": 1.591123342514038,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 191.0,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -1.834404706954956,
|
|
"logits/rejected": -1.859675645828247,
|
|
"logps/chosen": -0.3091299831867218,
|
|
"logps/rejected": -0.29531824588775635,
|
|
"loss": 96.41517639160156,
|
|
"loss/core": 96.41517639160156,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.204866886138916,
|
|
"rewards/margins": 2.250494956970215,
|
|
"rewards/rejected": 0.9543716311454773,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 138.0,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -1.7764365673065186,
|
|
"logits/rejected": -1.8405851125717163,
|
|
"logps/chosen": -0.30803143978118896,
|
|
"logps/rejected": -0.32817330956459045,
|
|
"loss": 98.6803207397461,
|
|
"loss/core": 98.6803207397461,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.861143708229065,
|
|
"rewards/margins": 0.8110074996948242,
|
|
"rewards/rejected": 1.0501360893249512,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 540.0,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -1.8258100748062134,
|
|
"logits/rejected": -1.7874729633331299,
|
|
"logps/chosen": -0.2907932698726654,
|
|
"logps/rejected": -0.3241172432899475,
|
|
"loss": 99.53788757324219,
|
|
"loss/core": 99.53788757324219,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4565396308898926,
|
|
"rewards/margins": 0.6172792315483093,
|
|
"rewards/rejected": 1.839260458946228,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 217.0,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -1.7738778591156006,
|
|
"logits/rejected": -1.847890853881836,
|
|
"logps/chosen": -0.28218382596969604,
|
|
"logps/rejected": -0.2760196328163147,
|
|
"loss": 96.96441650390625,
|
|
"loss/core": 96.96441650390625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.7287354469299316,
|
|
"rewards/margins": 1.59546959400177,
|
|
"rewards/rejected": 1.1332653760910034,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 592.0,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.7101322412490845,
|
|
"logits/rejected": -1.8211758136749268,
|
|
"logps/chosen": -0.297687828540802,
|
|
"logps/rejected": -0.27419671416282654,
|
|
"loss": 98.19453430175781,
|
|
"loss/core": 98.19453430175781,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.479161024093628,
|
|
"rewards/margins": 1.039329171180725,
|
|
"rewards/rejected": 1.4398319721221924,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 362.0,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -1.7563453912734985,
|
|
"logits/rejected": -1.736853837966919,
|
|
"logps/chosen": -0.2723637521266937,
|
|
"logps/rejected": -0.2854057848453522,
|
|
"loss": 98.63446807861328,
|
|
"loss/core": 98.63446807861328,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3016631603240967,
|
|
"rewards/margins": 0.736989438533783,
|
|
"rewards/rejected": 1.564673662185669,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 304.0,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -1.7407211065292358,
|
|
"logits/rejected": -1.863157868385315,
|
|
"logps/chosen": -0.2687687277793884,
|
|
"logps/rejected": -0.2672789990901947,
|
|
"loss": 96.38660430908203,
|
|
"loss/core": 96.38660430908203,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.118961811065674,
|
|
"rewards/margins": 2.259530782699585,
|
|
"rewards/rejected": 0.8594307899475098,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 97.93875423855252,
|
|
"train_runtime": 8489.6134,
|
|
"train_samples_per_second": 1.696,
|
|
"train_steps_per_second": 0.106
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|