3644 lines
135 KiB
JSON
3644 lines
135 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24268421530723572,
|
||
|
|
"drift/rejected_abs": 0.09689199179410934,
|
||
|
|
"epoch": 0.004777260241251642,
|
||
|
|
"grad_norm": 158.0,
|
||
|
|
"learning_rate": 1.111111111111111e-08,
|
||
|
|
"logits/chosen": -1.9987547397613525,
|
||
|
|
"logits/rejected": -2.0148520469665527,
|
||
|
|
"logps/chosen": -0.27876606583595276,
|
||
|
|
"logps/rejected": -0.2827395498752594,
|
||
|
|
"loss": 23.720317840576172,
|
||
|
|
"loss/core": 23.703845977783203,
|
||
|
|
"loss/preference_sft": 0.27876606583595276,
|
||
|
|
"loss/reference_anchor": 0.19172723591327667,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4264075756073,
|
||
|
|
"rewards/margins": 1.473706603050232,
|
||
|
|
"rewards/rejected": 0.9527010917663574,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1691116988658905,
|
||
|
|
"drift/rejected_abs": 0.1621948778629303,
|
||
|
|
"epoch": 0.009554520482503284,
|
||
|
|
"grad_norm": 1336.0,
|
||
|
|
"learning_rate": 2.5e-08,
|
||
|
|
"logits/chosen": -2.345815658569336,
|
||
|
|
"logits/rejected": -2.392836093902588,
|
||
|
|
"logps/chosen": -0.28935980796813965,
|
||
|
|
"logps/rejected": -0.2881030738353729,
|
||
|
|
"loss": 25.067256927490234,
|
||
|
|
"loss/core": 25.026386260986328,
|
||
|
|
"loss/preference_sft": 0.28935980796813965,
|
||
|
|
"loss/reference_anchor": 0.5160242915153503,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.6900745630264282,
|
||
|
|
"rewards/margins": 0.07324112951755524,
|
||
|
|
"rewards/rejected": 1.6168334484100342,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21343913674354553,
|
||
|
|
"drift/rejected_abs": 0.11456535756587982,
|
||
|
|
"epoch": 0.014331780723754926,
|
||
|
|
"grad_norm": 118.0,
|
||
|
|
"learning_rate": 3.888888888888889e-08,
|
||
|
|
"logits/chosen": -2.2311110496520996,
|
||
|
|
"logits/rejected": -2.1946258544921875,
|
||
|
|
"logps/chosen": -0.277534544467926,
|
||
|
|
"logps/rejected": -0.28368276357650757,
|
||
|
|
"loss": 24.18115997314453,
|
||
|
|
"loss/core": 24.16556167602539,
|
||
|
|
"loss/preference_sft": 0.277534544467926,
|
||
|
|
"loss/reference_anchor": 0.1802430897951126,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1329588890075684,
|
||
|
|
"rewards/margins": 0.9891046285629272,
|
||
|
|
"rewards/rejected": 1.1438543796539307,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19140727818012238,
|
||
|
|
"drift/rejected_abs": 0.08797380328178406,
|
||
|
|
"epoch": 0.01910904096500657,
|
||
|
|
"grad_norm": 104.5,
|
||
|
|
"learning_rate": 5.2777777777777776e-08,
|
||
|
|
"logits/chosen": -2.1776981353759766,
|
||
|
|
"logits/rejected": -2.2193965911865234,
|
||
|
|
"logps/chosen": -0.30152779817581177,
|
||
|
|
"logps/rejected": -0.2865992486476898,
|
||
|
|
"loss": 24.163585662841797,
|
||
|
|
"loss/core": 24.14507484436035,
|
||
|
|
"loss/preference_sft": 0.30152779817581177,
|
||
|
|
"loss/reference_anchor": 0.21664480865001678,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9140727519989014,
|
||
|
|
"rewards/margins": 1.0348074436187744,
|
||
|
|
"rewards/rejected": 0.8792654275894165,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3856329321861267,
|
||
|
|
"drift/rejected_abs": 0.16464610397815704,
|
||
|
|
"epoch": 0.02388630120625821,
|
||
|
|
"grad_norm": 86.5,
|
||
|
|
"learning_rate": 6.666666666666667e-08,
|
||
|
|
"logits/chosen": -2.0732712745666504,
|
||
|
|
"logits/rejected": -2.13415789604187,
|
||
|
|
"logps/chosen": -0.2700391411781311,
|
||
|
|
"logps/rejected": -0.2578909993171692,
|
||
|
|
"loss": 24.2296142578125,
|
||
|
|
"loss/core": 24.161888122558594,
|
||
|
|
"loss/preference_sft": 0.2700391411781311,
|
||
|
|
"loss/reference_anchor": 0.8760007619857788,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.8547744750976562,
|
||
|
|
"rewards/margins": 2.2091641426086426,
|
||
|
|
"rewards/rejected": 1.6456102132797241,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22750432789325714,
|
||
|
|
"drift/rejected_abs": 0.11023910343647003,
|
||
|
|
"epoch": 0.028663561447509853,
|
||
|
|
"grad_norm": 77.5,
|
||
|
|
"learning_rate": 8.055555555555555e-08,
|
||
|
|
"logits/chosen": -2.3493716716766357,
|
||
|
|
"logits/rejected": -2.3317930698394775,
|
||
|
|
"logps/chosen": -0.3004586100578308,
|
||
|
|
"logps/rejected": -0.3256753087043762,
|
||
|
|
"loss": 24.168228149414062,
|
||
|
|
"loss/core": 24.141149520874023,
|
||
|
|
"loss/preference_sft": 0.3004586100578308,
|
||
|
|
"loss/reference_anchor": 0.33102959394454956,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.2747082710266113,
|
||
|
|
"rewards/margins": 1.2620917558670044,
|
||
|
|
"rewards/rejected": 1.0126166343688965,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26583537459373474,
|
||
|
|
"drift/rejected_abs": 0.11596353352069855,
|
||
|
|
"epoch": 0.033440821688761495,
|
||
|
|
"grad_norm": 50.25,
|
||
|
|
"learning_rate": 9.444444444444444e-08,
|
||
|
|
"logits/chosen": -2.203460693359375,
|
||
|
|
"logits/rejected": -2.18483829498291,
|
||
|
|
"logps/chosen": -0.2677518129348755,
|
||
|
|
"logps/rejected": -0.26530081033706665,
|
||
|
|
"loss": 23.941822052001953,
|
||
|
|
"loss/core": 23.910053253173828,
|
||
|
|
"loss/preference_sft": 0.2677518129348755,
|
||
|
|
"loss/reference_anchor": 0.3968318998813629,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.6582398414611816,
|
||
|
|
"rewards/margins": 1.4989526271820068,
|
||
|
|
"rewards/rejected": 1.1592872142791748,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.44815701246261597,
|
||
|
|
"drift/rejected_abs": 0.16203448176383972,
|
||
|
|
"epoch": 0.03821808193001314,
|
||
|
|
"grad_norm": 151.0,
|
||
|
|
"learning_rate": 1.0833333333333334e-07,
|
||
|
|
"logits/chosen": -2.002789258956909,
|
||
|
|
"logits/rejected": -2.062656879425049,
|
||
|
|
"logps/chosen": -0.2923213839530945,
|
||
|
|
"logps/rejected": -0.2835318148136139,
|
||
|
|
"loss": 23.72269058227539,
|
||
|
|
"loss/core": 23.646509170532227,
|
||
|
|
"loss/preference_sft": 0.2923213839530945,
|
||
|
|
"loss/reference_anchor": 0.9865428805351257,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.481132984161377,
|
||
|
|
"rewards/margins": 2.8950064182281494,
|
||
|
|
"rewards/rejected": 1.5861257314682007,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20982900261878967,
|
||
|
|
"drift/rejected_abs": 0.08536230772733688,
|
||
|
|
"epoch": 0.04299534217126478,
|
||
|
|
"grad_norm": 65.5,
|
||
|
|
"learning_rate": 1.2222222222222222e-07,
|
||
|
|
"logits/chosen": -2.199631452560425,
|
||
|
|
"logits/rejected": -2.363292694091797,
|
||
|
|
"logps/chosen": -0.2886757254600525,
|
||
|
|
"logps/rejected": -0.2883760929107666,
|
||
|
|
"loss": 23.874040603637695,
|
||
|
|
"loss/core": 23.861454010009766,
|
||
|
|
"loss/preference_sft": 0.2886757254600525,
|
||
|
|
"loss/reference_anchor": 0.13895340263843536,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.0968005657196045,
|
||
|
|
"rewards/margins": 1.254421353340149,
|
||
|
|
"rewards/rejected": 0.8423792123794556,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2884393632411957,
|
||
|
|
"drift/rejected_abs": 0.1451389044523239,
|
||
|
|
"epoch": 0.04777260241251642,
|
||
|
|
"grad_norm": 402.0,
|
||
|
|
"learning_rate": 1.3611111111111108e-07,
|
||
|
|
"logits/chosen": -2.166757106781006,
|
||
|
|
"logits/rejected": -2.1628246307373047,
|
||
|
|
"logps/chosen": -0.28129029273986816,
|
||
|
|
"logps/rejected": -0.2850467562675476,
|
||
|
|
"loss": 24.05858612060547,
|
||
|
|
"loss/core": 24.029706954956055,
|
||
|
|
"loss/preference_sft": 0.28129029273986816,
|
||
|
|
"loss/reference_anchor": 0.3569650650024414,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8835978507995605,
|
||
|
|
"rewards/margins": 1.4343656301498413,
|
||
|
|
"rewards/rejected": 1.4492321014404297,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22632427513599396,
|
||
|
|
"drift/rejected_abs": 0.20729069411754608,
|
||
|
|
"epoch": 0.05254986265376806,
|
||
|
|
"grad_norm": 32.25,
|
||
|
|
"learning_rate": 1.5e-07,
|
||
|
|
"logits/chosen": -2.1963820457458496,
|
||
|
|
"logits/rejected": -2.1848561763763428,
|
||
|
|
"logps/chosen": -0.28860360383987427,
|
||
|
|
"logps/rejected": -0.28664013743400574,
|
||
|
|
"loss": 25.665725708007812,
|
||
|
|
"loss/core": 25.60601806640625,
|
||
|
|
"loss/preference_sft": 0.28860360383987427,
|
||
|
|
"loss/reference_anchor": 0.7672308087348938,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.262362003326416,
|
||
|
|
"rewards/margins": 0.20232732594013214,
|
||
|
|
"rewards/rejected": 2.0600345134735107,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21578149497509003,
|
||
|
|
"drift/rejected_abs": 0.1569838523864746,
|
||
|
|
"epoch": 0.057327122895019705,
|
||
|
|
"grad_norm": 1848.0,
|
||
|
|
"learning_rate": 1.6388888888888888e-07,
|
||
|
|
"logits/chosen": -2.0918967723846436,
|
||
|
|
"logits/rejected": -2.1219000816345215,
|
||
|
|
"logps/chosen": -0.27454960346221924,
|
||
|
|
"logps/rejected": -0.2835273742675781,
|
||
|
|
"loss": 24.56880760192871,
|
||
|
|
"loss/core": 24.553863525390625,
|
||
|
|
"loss/preference_sft": 0.27454960346221924,
|
||
|
|
"loss/reference_anchor": 0.1717853546142578,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1553452014923096,
|
||
|
|
"rewards/margins": 0.5968301892280579,
|
||
|
|
"rewards/rejected": 1.5585150718688965,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29738813638687134,
|
||
|
|
"drift/rejected_abs": 0.11811880022287369,
|
||
|
|
"epoch": 0.06210438313627135,
|
||
|
|
"grad_norm": 66.0,
|
||
|
|
"learning_rate": 1.7777777777777776e-07,
|
||
|
|
"logits/chosen": -1.962087869644165,
|
||
|
|
"logits/rejected": -2.0643563270568848,
|
||
|
|
"logps/chosen": -0.30899226665496826,
|
||
|
|
"logps/rejected": -0.3064674437046051,
|
||
|
|
"loss": 23.558834075927734,
|
||
|
|
"loss/core": 23.519794464111328,
|
||
|
|
"loss/preference_sft": 0.30899226665496826,
|
||
|
|
"loss/reference_anchor": 0.48962312936782837,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.9719326496124268,
|
||
|
|
"rewards/margins": 1.8016983270645142,
|
||
|
|
"rewards/rejected": 1.170233964920044,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3609868586063385,
|
||
|
|
"drift/rejected_abs": 0.228200763463974,
|
||
|
|
"epoch": 0.06688164337752299,
|
||
|
|
"grad_norm": 229.0,
|
||
|
|
"learning_rate": 1.9166666666666668e-07,
|
||
|
|
"logits/chosen": -1.9723434448242188,
|
||
|
|
"logits/rejected": -1.9571367502212524,
|
||
|
|
"logps/chosen": -0.3828391134738922,
|
||
|
|
"logps/rejected": -0.297024130821228,
|
||
|
|
"loss": 24.988039016723633,
|
||
|
|
"loss/core": 24.914230346679688,
|
||
|
|
"loss/preference_sft": 0.3828391134738922,
|
||
|
|
"loss/reference_anchor": 0.9458276629447937,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.6087005138397217,
|
||
|
|
"rewards/margins": 1.4257218837738037,
|
||
|
|
"rewards/rejected": 2.182979106903076,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2607436776161194,
|
||
|
|
"drift/rejected_abs": 0.1599571406841278,
|
||
|
|
"epoch": 0.07165890361877464,
|
||
|
|
"grad_norm": 56.5,
|
||
|
|
"learning_rate": 2.0555555555555553e-07,
|
||
|
|
"logits/chosen": -2.0862269401550293,
|
||
|
|
"logits/rejected": -2.050337553024292,
|
||
|
|
"logps/chosen": -0.31514787673950195,
|
||
|
|
"logps/rejected": -0.28866785764694214,
|
||
|
|
"loss": 24.35919189453125,
|
||
|
|
"loss/core": 24.329198837280273,
|
||
|
|
"loss/preference_sft": 0.31514787673950195,
|
||
|
|
"loss/reference_anchor": 0.36842408776283264,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.6053974628448486,
|
||
|
|
"rewards/margins": 1.00749671459198,
|
||
|
|
"rewards/rejected": 1.5979008674621582,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3240804970264435,
|
||
|
|
"drift/rejected_abs": 0.12827441096305847,
|
||
|
|
"epoch": 0.07643616386002627,
|
||
|
|
"grad_norm": 55.5,
|
||
|
|
"learning_rate": 2.1944444444444442e-07,
|
||
|
|
"logits/chosen": -2.0698435306549072,
|
||
|
|
"logits/rejected": -2.1897106170654297,
|
||
|
|
"logps/chosen": -0.26063475012779236,
|
||
|
|
"logps/rejected": -0.2740657925605774,
|
||
|
|
"loss": 23.68960189819336,
|
||
|
|
"loss/core": 23.653255462646484,
|
||
|
|
"loss/preference_sft": 0.26063475012779236,
|
||
|
|
"loss/reference_anchor": 0.45853161811828613,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.2406482696533203,
|
||
|
|
"rewards/margins": 1.9891496896743774,
|
||
|
|
"rewards/rejected": 1.2514985799789429,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3498837351799011,
|
||
|
|
"drift/rejected_abs": 0.13262492418289185,
|
||
|
|
"epoch": 0.08121342410127792,
|
||
|
|
"grad_norm": 79.0,
|
||
|
|
"learning_rate": 2.3333333333333333e-07,
|
||
|
|
"logits/chosen": -2.189903497695923,
|
||
|
|
"logits/rejected": -2.2383172512054443,
|
||
|
|
"logps/chosen": -0.3075157105922699,
|
||
|
|
"logps/rejected": -0.2782510817050934,
|
||
|
|
"loss": 23.684247970581055,
|
||
|
|
"loss/core": 23.61785125732422,
|
||
|
|
"loss/preference_sft": 0.3075157105922699,
|
||
|
|
"loss/reference_anchor": 0.8545700311660767,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.498150587081909,
|
||
|
|
"rewards/margins": 2.172797918319702,
|
||
|
|
"rewards/rejected": 1.3253527879714966,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17537356913089752,
|
||
|
|
"drift/rejected_abs": 0.08442483097314835,
|
||
|
|
"epoch": 0.08599068434252956,
|
||
|
|
"grad_norm": 31.875,
|
||
|
|
"learning_rate": 2.4722222222222224e-07,
|
||
|
|
"logits/chosen": -2.4203712940216064,
|
||
|
|
"logits/rejected": -2.4085536003112793,
|
||
|
|
"logps/chosen": -0.28136521577835083,
|
||
|
|
"logps/rejected": -0.296181857585907,
|
||
|
|
"loss": 24.193899154663086,
|
||
|
|
"loss/core": 24.180316925048828,
|
||
|
|
"loss/preference_sft": 0.28136521577835083,
|
||
|
|
"loss/reference_anchor": 0.15298345685005188,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7537355422973633,
|
||
|
|
"rewards/margins": 0.9103192090988159,
|
||
|
|
"rewards/rejected": 0.8434165716171265,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3204111158847809,
|
||
|
|
"drift/rejected_abs": 0.21080467104911804,
|
||
|
|
"epoch": 0.09076794458378121,
|
||
|
|
"grad_norm": 118.0,
|
||
|
|
"learning_rate": 2.6111111111111113e-07,
|
||
|
|
"logits/chosen": -2.0531086921691895,
|
||
|
|
"logits/rejected": -2.108947992324829,
|
||
|
|
"logps/chosen": -0.267969012260437,
|
||
|
|
"logps/rejected": -0.27856558561325073,
|
||
|
|
"loss": 24.32411766052246,
|
||
|
|
"loss/core": 24.288156509399414,
|
||
|
|
"loss/preference_sft": 0.267969012260437,
|
||
|
|
"loss/reference_anchor": 0.4526856541633606,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.2040367126464844,
|
||
|
|
"rewards/margins": 1.0959895849227905,
|
||
|
|
"rewards/rejected": 2.1080470085144043,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2465881109237671,
|
||
|
|
"drift/rejected_abs": 0.1286160796880722,
|
||
|
|
"epoch": 0.09554520482503284,
|
||
|
|
"grad_norm": 73.5,
|
||
|
|
"learning_rate": 2.75e-07,
|
||
|
|
"logits/chosen": -2.2330098152160645,
|
||
|
|
"logits/rejected": -2.2795863151550293,
|
||
|
|
"logps/chosen": -0.2569820284843445,
|
||
|
|
"logps/rejected": -0.2466917484998703,
|
||
|
|
"loss": 23.988306045532227,
|
||
|
|
"loss/core": 23.972742080688477,
|
||
|
|
"loss/preference_sft": 0.2569820284843445,
|
||
|
|
"loss/reference_anchor": 0.18182215094566345,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.4647648334503174,
|
||
|
|
"rewards/margins": 1.1787031888961792,
|
||
|
|
"rewards/rejected": 1.2860618829727173,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2001236379146576,
|
||
|
|
"drift/rejected_abs": 0.079637810587883,
|
||
|
|
"epoch": 0.10032246506628449,
|
||
|
|
"grad_norm": 50.25,
|
||
|
|
"learning_rate": 2.8888888888888885e-07,
|
||
|
|
"logits/chosen": -2.3382439613342285,
|
||
|
|
"logits/rejected": -2.424198865890503,
|
||
|
|
"logps/chosen": -0.2838006615638733,
|
||
|
|
"logps/rejected": -0.28019005060195923,
|
||
|
|
"loss": 24.08165168762207,
|
||
|
|
"loss/core": 24.064653396606445,
|
||
|
|
"loss/preference_sft": 0.2838006615638733,
|
||
|
|
"loss/reference_anchor": 0.1982627958059311,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.0012366771698,
|
||
|
|
"rewards/margins": 1.207162618637085,
|
||
|
|
"rewards/rejected": 0.7940739393234253,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4472831189632416,
|
||
|
|
"drift/rejected_abs": 0.18904665112495422,
|
||
|
|
"epoch": 0.10509972530753613,
|
||
|
|
"grad_norm": 71.0,
|
||
|
|
"learning_rate": 3.0277777777777773e-07,
|
||
|
|
"logits/chosen": -1.9077485799789429,
|
||
|
|
"logits/rejected": -1.8608009815216064,
|
||
|
|
"logps/chosen": -0.3383413255214691,
|
||
|
|
"logps/rejected": -0.2978266477584839,
|
||
|
|
"loss": 23.436580657958984,
|
||
|
|
"loss/core": 23.36850357055664,
|
||
|
|
"loss/preference_sft": 0.3383413255214691,
|
||
|
|
"loss/reference_anchor": 0.8738870620727539,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.471864223480225,
|
||
|
|
"rewards/margins": 2.584223508834839,
|
||
|
|
"rewards/rejected": 1.8876409530639648,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17792600393295288,
|
||
|
|
"drift/rejected_abs": 0.14627966284751892,
|
||
|
|
"epoch": 0.10987698554878778,
|
||
|
|
"grad_norm": 334.0,
|
||
|
|
"learning_rate": 3.166666666666666e-07,
|
||
|
|
"logits/chosen": -2.088899850845337,
|
||
|
|
"logits/rejected": -2.032494068145752,
|
||
|
|
"logps/chosen": -0.30003082752227783,
|
||
|
|
"logps/rejected": -0.2972942888736725,
|
||
|
|
"loss": 24.94771957397461,
|
||
|
|
"loss/core": 24.928913116455078,
|
||
|
|
"loss/preference_sft": 0.30003082752227783,
|
||
|
|
"loss/reference_anchor": 0.22078463435173035,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7790625095367432,
|
||
|
|
"rewards/margins": 0.31695252656936646,
|
||
|
|
"rewards/rejected": 1.4621098041534424,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18117384612560272,
|
||
|
|
"drift/rejected_abs": 0.11267610639333725,
|
||
|
|
"epoch": 0.11465424579003941,
|
||
|
|
"grad_norm": 768.0,
|
||
|
|
"learning_rate": 3.3055555555555556e-07,
|
||
|
|
"logits/chosen": -2.0376081466674805,
|
||
|
|
"logits/rejected": -2.0747933387756348,
|
||
|
|
"logps/chosen": -0.3019946813583374,
|
||
|
|
"logps/rejected": -0.29592204093933105,
|
||
|
|
"loss": 24.376644134521484,
|
||
|
|
"loss/core": 24.367816925048828,
|
||
|
|
"loss/preference_sft": 0.3019946813583374,
|
||
|
|
"loss/reference_anchor": 0.08752311021089554,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8115949630737305,
|
||
|
|
"rewards/margins": 0.6873809099197388,
|
||
|
|
"rewards/rejected": 1.1242139339447021,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20648498833179474,
|
||
|
|
"drift/rejected_abs": 0.10197492688894272,
|
||
|
|
"epoch": 0.11943150603129106,
|
||
|
|
"grad_norm": 50.75,
|
||
|
|
"learning_rate": 3.4444444444444444e-07,
|
||
|
|
"logits/chosen": -2.1082651615142822,
|
||
|
|
"logits/rejected": -2.139118194580078,
|
||
|
|
"logps/chosen": -0.29931074380874634,
|
||
|
|
"logps/rejected": -0.30651599168777466,
|
||
|
|
"loss": 24.086589813232422,
|
||
|
|
"loss/core": 24.07431983947754,
|
||
|
|
"loss/preference_sft": 0.29931074380874634,
|
||
|
|
"loss/reference_anchor": 0.13366781175136566,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.0623745918273926,
|
||
|
|
"rewards/margins": 1.0441718101501465,
|
||
|
|
"rewards/rejected": 1.018202543258667,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18083539605140686,
|
||
|
|
"drift/rejected_abs": 0.08944060653448105,
|
||
|
|
"epoch": 0.1242087662725427,
|
||
|
|
"grad_norm": 100.5,
|
||
|
|
"learning_rate": 3.583333333333333e-07,
|
||
|
|
"logits/chosen": -2.0009732246398926,
|
||
|
|
"logits/rejected": -2.0936858654022217,
|
||
|
|
"logps/chosen": -0.27988457679748535,
|
||
|
|
"logps/rejected": -0.278131902217865,
|
||
|
|
"loss": 24.129959106445312,
|
||
|
|
"loss/core": 24.122276306152344,
|
||
|
|
"loss/preference_sft": 0.27988457679748535,
|
||
|
|
"loss/reference_anchor": 0.07443337142467499,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.8083540201187134,
|
||
|
|
"rewards/margins": 0.9148699641227722,
|
||
|
|
"rewards/rejected": 0.8934842348098755,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.33803224563598633,
|
||
|
|
"drift/rejected_abs": 0.13368059694766998,
|
||
|
|
"epoch": 0.12898602651379434,
|
||
|
|
"grad_norm": 134.0,
|
||
|
|
"learning_rate": 3.722222222222222e-07,
|
||
|
|
"logits/chosen": -2.0908658504486084,
|
||
|
|
"logits/rejected": -2.1650338172912598,
|
||
|
|
"logps/chosen": -0.3015690743923187,
|
||
|
|
"logps/rejected": -0.2998407483100891,
|
||
|
|
"loss": 23.31682777404785,
|
||
|
|
"loss/core": 23.28109359741211,
|
||
|
|
"loss/preference_sft": 0.3015690743923187,
|
||
|
|
"loss/reference_anchor": 0.446286141872406,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.3794403076171875,
|
||
|
|
"rewards/margins": 2.081610918045044,
|
||
|
|
"rewards/rejected": 1.2978298664093018,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16290757060050964,
|
||
|
|
"drift/rejected_abs": 0.07148541510105133,
|
||
|
|
"epoch": 0.13376328675504598,
|
||
|
|
"grad_norm": 60.25,
|
||
|
|
"learning_rate": 3.861111111111111e-07,
|
||
|
|
"logits/chosen": -2.4541397094726562,
|
||
|
|
"logits/rejected": -2.5143344402313232,
|
||
|
|
"logps/chosen": -0.28250986337661743,
|
||
|
|
"logps/rejected": -0.2895656228065491,
|
||
|
|
"loss": 24.135583877563477,
|
||
|
|
"loss/core": 24.1291446685791,
|
||
|
|
"loss/preference_sft": 0.28250986337661743,
|
||
|
|
"loss/reference_anchor": 0.05761883780360222,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.6281633377075195,
|
||
|
|
"rewards/margins": 0.9143427610397339,
|
||
|
|
"rewards/rejected": 0.7138205170631409,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3603426516056061,
|
||
|
|
"drift/rejected_abs": 0.17863936722278595,
|
||
|
|
"epoch": 0.13854054699629761,
|
||
|
|
"grad_norm": 96.0,
|
||
|
|
"learning_rate": 4e-07,
|
||
|
|
"logits/chosen": -1.77964186668396,
|
||
|
|
"logits/rejected": -1.8101189136505127,
|
||
|
|
"logps/chosen": -0.30144834518432617,
|
||
|
|
"logps/rejected": -0.30214372277259827,
|
||
|
|
"loss": 23.7529239654541,
|
||
|
|
"loss/core": 23.6727352142334,
|
||
|
|
"loss/preference_sft": 0.30144834518432617,
|
||
|
|
"loss/reference_anchor": 1.0389857292175293,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.6030845642089844,
|
||
|
|
"rewards/margins": 1.8218176364898682,
|
||
|
|
"rewards/rejected": 1.7812671661376953,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36440587043762207,
|
||
|
|
"drift/rejected_abs": 0.1463252753019333,
|
||
|
|
"epoch": 0.14331780723754928,
|
||
|
|
"grad_norm": 98.5,
|
||
|
|
"learning_rate": 4.1388888888888887e-07,
|
||
|
|
"logits/chosen": -2.2338650226593018,
|
||
|
|
"logits/rejected": -2.282841444015503,
|
||
|
|
"logps/chosen": -0.31720206141471863,
|
||
|
|
"logps/rejected": -0.3081175684928894,
|
||
|
|
"loss": 23.425756454467773,
|
||
|
|
"loss/core": 23.36142349243164,
|
||
|
|
"loss/preference_sft": 0.31720206141471863,
|
||
|
|
"loss/reference_anchor": 0.8260464668273926,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.640651226043701,
|
||
|
|
"rewards/margins": 2.182239055633545,
|
||
|
|
"rewards/rejected": 1.4584124088287354,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28685861825942993,
|
||
|
|
"drift/rejected_abs": 0.07811577618122101,
|
||
|
|
"epoch": 0.1480950674788009,
|
||
|
|
"grad_norm": 77.5,
|
||
|
|
"learning_rate": 4.2777777777777775e-07,
|
||
|
|
"logits/chosen": -2.0995328426361084,
|
||
|
|
"logits/rejected": -2.177826166152954,
|
||
|
|
"logps/chosen": -0.2826915383338928,
|
||
|
|
"logps/rejected": -0.29879477620124817,
|
||
|
|
"loss": 23.43790054321289,
|
||
|
|
"loss/core": 23.408916473388672,
|
||
|
|
"loss/preference_sft": 0.2826915383338928,
|
||
|
|
"loss/reference_anchor": 0.3581943213939667,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8685860633850098,
|
||
|
|
"rewards/margins": 2.106802225112915,
|
||
|
|
"rewards/rejected": 0.7617841958999634,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31482672691345215,
|
||
|
|
"drift/rejected_abs": 0.1407286822795868,
|
||
|
|
"epoch": 0.15287232772005255,
|
||
|
|
"grad_norm": 200.0,
|
||
|
|
"learning_rate": 4.4166666666666664e-07,
|
||
|
|
"logits/chosen": -2.05705189704895,
|
||
|
|
"logits/rejected": -2.0315186977386475,
|
||
|
|
"logps/chosen": -0.31071725487709045,
|
||
|
|
"logps/rejected": -0.31473129987716675,
|
||
|
|
"loss": 24.094379425048828,
|
||
|
|
"loss/core": 24.039676666259766,
|
||
|
|
"loss/preference_sft": 0.31071725487709045,
|
||
|
|
"loss/reference_anchor": 0.698303759098053,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.1482670307159424,
|
||
|
|
"rewards/margins": 1.7411534786224365,
|
||
|
|
"rewards/rejected": 1.4071136713027954,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1602366864681244,
|
||
|
|
"drift/rejected_abs": 0.07925491034984589,
|
||
|
|
"epoch": 0.15764958796130418,
|
||
|
|
"grad_norm": 80.5,
|
||
|
|
"learning_rate": 4.555555555555555e-07,
|
||
|
|
"logits/chosen": -2.223989725112915,
|
||
|
|
"logits/rejected": -2.1556718349456787,
|
||
|
|
"logps/chosen": -0.3111506998538971,
|
||
|
|
"logps/rejected": -0.30567336082458496,
|
||
|
|
"loss": 24.250789642333984,
|
||
|
|
"loss/core": 24.241710662841797,
|
||
|
|
"loss/preference_sft": 0.3111506998538971,
|
||
|
|
"loss/reference_anchor": 0.08993285894393921,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.601798415184021,
|
||
|
|
"rewards/margins": 0.8125208020210266,
|
||
|
|
"rewards/rejected": 0.7892775535583496,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2597414255142212,
|
||
|
|
"drift/rejected_abs": 0.2784227728843689,
|
||
|
|
"epoch": 0.16242684820255585,
|
||
|
|
"grad_norm": 81.5,
|
||
|
|
"learning_rate": 4.694444444444444e-07,
|
||
|
|
"logits/chosen": -2.187836170196533,
|
||
|
|
"logits/rejected": -2.132575511932373,
|
||
|
|
"logps/chosen": -0.27488964796066284,
|
||
|
|
"logps/rejected": -0.2585277855396271,
|
||
|
|
"loss": 26.337051391601562,
|
||
|
|
"loss/core": 26.263935089111328,
|
||
|
|
"loss/preference_sft": 0.27488964796066284,
|
||
|
|
"loss/reference_anchor": 0.9473751783370972,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.59535551071167,
|
||
|
|
"rewards/margins": -0.186926007270813,
|
||
|
|
"rewards/rejected": 2.7822816371917725,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2583083212375641,
|
||
|
|
"drift/rejected_abs": 0.12779515981674194,
|
||
|
|
"epoch": 0.16720410844380748,
|
||
|
|
"grad_norm": 372.0,
|
||
|
|
"learning_rate": 4.833333333333333e-07,
|
||
|
|
"logits/chosen": -2.1770424842834473,
|
||
|
|
"logits/rejected": -2.216740608215332,
|
||
|
|
"logps/chosen": -0.2869277000427246,
|
||
|
|
"logps/rejected": -0.30118730664253235,
|
||
|
|
"loss": 23.88767433166504,
|
||
|
|
"loss/core": 23.843341827392578,
|
||
|
|
"loss/preference_sft": 0.2869277000427246,
|
||
|
|
"loss/reference_anchor": 0.5624145865440369,
|
||
|
|
"rewards/accuracies": 0.987500011920929,
|
||
|
|
"rewards/chosen": 2.583082914352417,
|
||
|
|
"rewards/margins": 1.3182199001312256,
|
||
|
|
"rewards/rejected": 1.2648632526397705,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3239268660545349,
|
||
|
|
"drift/rejected_abs": 0.1492597758769989,
|
||
|
|
"epoch": 0.17198136868505912,
|
||
|
|
"grad_norm": 280.0,
|
||
|
|
"learning_rate": 4.972222222222222e-07,
|
||
|
|
"logits/chosen": -2.201796054840088,
|
||
|
|
"logits/rejected": -2.111999988555908,
|
||
|
|
"logps/chosen": -0.27172645926475525,
|
||
|
|
"logps/rejected": -0.2618543803691864,
|
||
|
|
"loss": 23.988094329833984,
|
||
|
|
"loss/core": 23.94233512878418,
|
||
|
|
"loss/preference_sft": 0.27172645926475525,
|
||
|
|
"loss/reference_anchor": 0.5829019546508789,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.2370712757110596,
|
||
|
|
"rewards/margins": 1.7461532354354858,
|
||
|
|
"rewards/rejected": 1.4909178018569946,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.10453528165817261,
|
||
|
|
"drift/rejected_abs": 0.06307251751422882,
|
||
|
|
"epoch": 0.17675862892631075,
|
||
|
|
"grad_norm": 62.0,
|
||
|
|
"learning_rate": 4.999619237890978e-07,
|
||
|
|
"logits/chosen": -2.206681966781616,
|
||
|
|
"logits/rejected": -2.1635756492614746,
|
||
|
|
"logps/chosen": -0.2884823679924011,
|
||
|
|
"logps/rejected": -0.29190367460250854,
|
||
|
|
"loss": 24.60689926147461,
|
||
|
|
"loss/core": 24.602405548095703,
|
||
|
|
"loss/preference_sft": 0.2884823679924011,
|
||
|
|
"loss/reference_anchor": 0.031054329127073288,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.0448436737060547,
|
||
|
|
"rewards/margins": 0.41608095169067383,
|
||
|
|
"rewards/rejected": 0.6287627816200256,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28438663482666016,
|
||
|
|
"drift/rejected_abs": 0.12391813099384308,
|
||
|
|
"epoch": 0.18153588916756241,
|
||
|
|
"grad_norm": 125.0,
|
||
|
|
"learning_rate": 4.998072590601808e-07,
|
||
|
|
"logits/chosen": -2.2966725826263428,
|
||
|
|
"logits/rejected": -2.2743964195251465,
|
||
|
|
"logps/chosen": -0.28313106298446655,
|
||
|
|
"logps/rejected": -0.28445905447006226,
|
||
|
|
"loss": 23.94525718688965,
|
||
|
|
"loss/core": 23.900707244873047,
|
||
|
|
"loss/preference_sft": 0.28313106298446655,
|
||
|
|
"loss/reference_anchor": 0.5657359957695007,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.842867374420166,
|
||
|
|
"rewards/margins": 1.6057891845703125,
|
||
|
|
"rewards/rejected": 1.2370779514312744,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26715975999832153,
|
||
|
|
"drift/rejected_abs": 0.09772941470146179,
|
||
|
|
"epoch": 0.18631314940881405,
|
||
|
|
"grad_norm": 67.5,
|
||
|
|
"learning_rate": 4.995336996054667e-07,
|
||
|
|
"logits/chosen": -1.9860740900039673,
|
||
|
|
"logits/rejected": -2.006164073944092,
|
||
|
|
"logps/chosen": -0.29072457551956177,
|
||
|
|
"logps/rejected": -0.28935790061950684,
|
||
|
|
"loss": 24.14964485168457,
|
||
|
|
"loss/core": 24.105886459350586,
|
||
|
|
"loss/preference_sft": 0.29072457551956177,
|
||
|
|
"loss/reference_anchor": 0.5543695092201233,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.6712136268615723,
|
||
|
|
"rewards/margins": 1.701263427734375,
|
||
|
|
"rewards/rejected": 0.9699504971504211,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26089510321617126,
|
||
|
|
"drift/rejected_abs": 0.12243443727493286,
|
||
|
|
"epoch": 0.19109040965006568,
|
||
|
|
"grad_norm": 75.0,
|
||
|
|
"learning_rate": 4.991413756244404e-07,
|
||
|
|
"logits/chosen": -1.8967984914779663,
|
||
|
|
"logits/rejected": -1.9376294612884521,
|
||
|
|
"logps/chosen": -0.3242345452308655,
|
||
|
|
"logps/rejected": -0.3163239061832428,
|
||
|
|
"loss": 23.804271697998047,
|
||
|
|
"loss/core": 23.7833309173584,
|
||
|
|
"loss/preference_sft": 0.3242345452308655,
|
||
|
|
"loss/reference_anchor": 0.2467818558216095,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.6078362464904785,
|
||
|
|
"rewards/margins": 1.3843178749084473,
|
||
|
|
"rewards/rejected": 1.2235184907913208,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.328901469707489,
|
||
|
|
"drift/rejected_abs": 0.2119167149066925,
|
||
|
|
"epoch": 0.19586766989131732,
|
||
|
|
"grad_norm": 50.75,
|
||
|
|
"learning_rate": 4.986304738420683e-07,
|
||
|
|
"logits/chosen": -2.1179518699645996,
|
||
|
|
"logits/rejected": -2.169663667678833,
|
||
|
|
"logps/chosen": -0.25019803643226624,
|
||
|
|
"logps/rejected": -0.2696530222892761,
|
||
|
|
"loss": 24.55345344543457,
|
||
|
|
"loss/core": 24.50376319885254,
|
||
|
|
"loss/preference_sft": 0.25019803643226624,
|
||
|
|
"loss/reference_anchor": 0.6374753713607788,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.288543224334717,
|
||
|
|
"rewards/margins": 1.1873959302902222,
|
||
|
|
"rewards/rejected": 2.101147413253784,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2828282415866852,
|
||
|
|
"drift/rejected_abs": 0.1485763043165207,
|
||
|
|
"epoch": 0.20064493013256898,
|
||
|
|
"grad_norm": 121.0,
|
||
|
|
"learning_rate": 4.980012374199287e-07,
|
||
|
|
"logits/chosen": -2.1135048866271973,
|
||
|
|
"logits/rejected": -2.1378347873687744,
|
||
|
|
"logps/chosen": -0.3129284381866455,
|
||
|
|
"logps/rejected": -0.32191547751426697,
|
||
|
|
"loss": 23.932619094848633,
|
||
|
|
"loss/core": 23.88772964477539,
|
||
|
|
"loss/preference_sft": 0.3129284381866455,
|
||
|
|
"loss/reference_anchor": 0.5672147870063782,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8268136978149414,
|
||
|
|
"rewards/margins": 1.3436048030853271,
|
||
|
|
"rewards/rejected": 1.4832088947296143,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3194449245929718,
|
||
|
|
"drift/rejected_abs": 0.1719251424074173,
|
||
|
|
"epoch": 0.20542219037382062,
|
||
|
|
"grad_norm": 133.0,
|
||
|
|
"learning_rate": 4.972539658404792e-07,
|
||
|
|
"logits/chosen": -2.0891597270965576,
|
||
|
|
"logits/rejected": -2.0813040733337402,
|
||
|
|
"logps/chosen": -0.2754015326499939,
|
||
|
|
"logps/rejected": -0.2838408946990967,
|
||
|
|
"loss": 24.59800910949707,
|
||
|
|
"loss/core": 24.53702735900879,
|
||
|
|
"loss/preference_sft": 0.2754015326499939,
|
||
|
|
"loss/reference_anchor": 0.7855226397514343,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.1921732425689697,
|
||
|
|
"rewards/margins": 1.485547661781311,
|
||
|
|
"rewards/rejected": 1.7066256999969482,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18876484036445618,
|
||
|
|
"drift/rejected_abs": 0.10738082975149155,
|
||
|
|
"epoch": 0.21019945061507225,
|
||
|
|
"grad_norm": 138.0,
|
||
|
|
"learning_rate": 4.963890147645194e-07,
|
||
|
|
"logits/chosen": -2.1035685539245605,
|
||
|
|
"logits/rejected": -2.017023801803589,
|
||
|
|
"logps/chosen": -0.3024953007698059,
|
||
|
|
"logps/rejected": -0.30150723457336426,
|
||
|
|
"loss": 24.240825653076172,
|
||
|
|
"loss/core": 24.230350494384766,
|
||
|
|
"loss/preference_sft": 0.3024953007698059,
|
||
|
|
"loss/reference_anchor": 0.1094045639038086,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.8857002258300781,
|
||
|
|
"rewards/margins": 0.813431441783905,
|
||
|
|
"rewards/rejected": 1.0722688436508179,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36280855536460876,
|
||
|
|
"drift/rejected_abs": 0.14171245694160461,
|
||
|
|
"epoch": 0.2149767108563239,
|
||
|
|
"grad_norm": 68.0,
|
||
|
|
"learning_rate": 4.95406795861916e-07,
|
||
|
|
"logits/chosen": -2.189195156097412,
|
||
|
|
"logits/rejected": -2.1777374744415283,
|
||
|
|
"logps/chosen": -0.29006287455558777,
|
||
|
|
"logps/rejected": -0.2748886048793793,
|
||
|
|
"loss": 23.25436782836914,
|
||
|
|
"loss/core": 23.220571517944336,
|
||
|
|
"loss/preference_sft": 0.29006287455558777,
|
||
|
|
"loss/reference_anchor": 0.4215586185455322,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.6274166107177734,
|
||
|
|
"rewards/margins": 2.212296485900879,
|
||
|
|
"rewards/rejected": 1.4151195287704468,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18566292524337769,
|
||
|
|
"drift/rejected_abs": 0.07588974386453629,
|
||
|
|
"epoch": 0.21975397109757555,
|
||
|
|
"grad_norm": 828.0,
|
||
|
|
"learning_rate": 4.943077766156697e-07,
|
||
|
|
"logits/chosen": -2.2765140533447266,
|
||
|
|
"logits/rejected": -2.2845187187194824,
|
||
|
|
"logps/chosen": -0.2816498279571533,
|
||
|
|
"logps/rejected": -0.30764302611351013,
|
||
|
|
"loss": 24.015369415283203,
|
||
|
|
"loss/core": 24.00626564025879,
|
||
|
|
"loss/preference_sft": 0.2816498279571533,
|
||
|
|
"loss/reference_anchor": 0.09326620399951935,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.8551431894302368,
|
||
|
|
"rewards/margins": 1.0997159481048584,
|
||
|
|
"rewards/rejected": 0.7554270625114441,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19052401185035706,
|
||
|
|
"drift/rejected_abs": 0.13283082842826843,
|
||
|
|
"epoch": 0.2245312313388272,
|
||
|
|
"grad_norm": 75.0,
|
||
|
|
"learning_rate": 4.930924800994191e-07,
|
||
|
|
"logits/chosen": -2.0162127017974854,
|
||
|
|
"logits/rejected": -2.050363302230835,
|
||
|
|
"logps/chosen": -0.2928079068660736,
|
||
|
|
"logps/rejected": -0.28698280453681946,
|
||
|
|
"loss": 24.55010414123535,
|
||
|
|
"loss/core": 24.53103256225586,
|
||
|
|
"loss/preference_sft": 0.2928079068660736,
|
||
|
|
"loss/reference_anchor": 0.22502753138542175,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.9030735492706299,
|
||
|
|
"rewards/margins": 0.5786088705062866,
|
||
|
|
"rewards/rejected": 1.3244645595550537,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1768554449081421,
|
||
|
|
"drift/rejected_abs": 0.10264088213443756,
|
||
|
|
"epoch": 0.22930849158007882,
|
||
|
|
"grad_norm": 88.0,
|
||
|
|
"learning_rate": 4.917614847284858e-07,
|
||
|
|
"logits/chosen": -2.0287160873413086,
|
||
|
|
"logits/rejected": -2.0479371547698975,
|
||
|
|
"logps/chosen": -0.29354214668273926,
|
||
|
|
"logps/rejected": -0.29428401589393616,
|
||
|
|
"loss": 24.292566299438477,
|
||
|
|
"loss/core": 24.284984588623047,
|
||
|
|
"loss/preference_sft": 0.29354214668273926,
|
||
|
|
"loss/reference_anchor": 0.07169236242771149,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.767877221107483,
|
||
|
|
"rewards/margins": 0.7428776621818542,
|
||
|
|
"rewards/rejected": 1.0249996185302734,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.35989120602607727,
|
||
|
|
"drift/rejected_abs": 0.1157422810792923,
|
||
|
|
"epoch": 0.23408575182133046,
|
||
|
|
"grad_norm": 1056.0,
|
||
|
|
"learning_rate": 4.903154239845797e-07,
|
||
|
|
"logits/chosen": -2.0682170391082764,
|
||
|
|
"logits/rejected": -2.0564630031585693,
|
||
|
|
"logps/chosen": -0.2781009078025818,
|
||
|
|
"logps/rejected": -0.2913815677165985,
|
||
|
|
"loss": 23.59634017944336,
|
||
|
|
"loss/core": 23.54959487915039,
|
||
|
|
"loss/preference_sft": 0.2781009078025818,
|
||
|
|
"loss/reference_anchor": 0.5954445600509644,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.5985991954803467,
|
||
|
|
"rewards/margins": 2.4494547843933105,
|
||
|
|
"rewards/rejected": 1.1491445302963257,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2339925318956375,
|
||
|
|
"drift/rejected_abs": 0.10722045600414276,
|
||
|
|
"epoch": 0.23886301206258212,
|
||
|
|
"grad_norm": 101.0,
|
||
|
|
"learning_rate": 4.887549861142967e-07,
|
||
|
|
"logits/chosen": -1.9105064868927002,
|
||
|
|
"logits/rejected": -1.9906113147735596,
|
||
|
|
"logps/chosen": -0.2999528646469116,
|
||
|
|
"logps/rejected": -0.29507893323898315,
|
||
|
|
"loss": 23.889766693115234,
|
||
|
|
"loss/core": 23.876558303833008,
|
||
|
|
"loss/preference_sft": 0.2999528646469116,
|
||
|
|
"loss/reference_anchor": 0.1460881531238556,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.337829351425171,
|
||
|
|
"rewards/margins": 1.2744026184082031,
|
||
|
|
"rewards/rejected": 1.0634267330169678,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24554595351219177,
|
||
|
|
"drift/rejected_abs": 0.16090847551822662,
|
||
|
|
"epoch": 0.24364027230383375,
|
||
|
|
"grad_norm": 64.0,
|
||
|
|
"learning_rate": 4.870809138015498e-07,
|
||
|
|
"logits/chosen": -2.220195770263672,
|
||
|
|
"logits/rejected": -2.2151570320129395,
|
||
|
|
"logps/chosen": -0.2845707833766937,
|
||
|
|
"logps/rejected": -0.28703150153160095,
|
||
|
|
"loss": 24.28571128845215,
|
||
|
|
"loss/core": 24.260061264038086,
|
||
|
|
"loss/preference_sft": 0.2845707833766937,
|
||
|
|
"loss/reference_anchor": 0.31353074312210083,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4550528526306152,
|
||
|
|
"rewards/margins": 0.8476669192314148,
|
||
|
|
"rewards/rejected": 1.6073859930038452,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2403588742017746,
|
||
|
|
"drift/rejected_abs": 0.15879188477993011,
|
||
|
|
"epoch": 0.2484175325450854,
|
||
|
|
"grad_norm": 35.25,
|
||
|
|
"learning_rate": 4.852940038140927e-07,
|
||
|
|
"logits/chosen": -2.1549439430236816,
|
||
|
|
"logits/rejected": -2.227227210998535,
|
||
|
|
"logps/chosen": -0.28740060329437256,
|
||
|
|
"logps/rejected": -0.27014413475990295,
|
||
|
|
"loss": 24.529048919677734,
|
||
|
|
"loss/core": 24.506467819213867,
|
||
|
|
"loss/preference_sft": 0.28740060329437256,
|
||
|
|
"loss/reference_anchor": 0.2723291516304016,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.403057336807251,
|
||
|
|
"rewards/margins": 0.8171398043632507,
|
||
|
|
"rewards/rejected": 1.5859174728393555,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12319445610046387,
|
||
|
|
"drift/rejected_abs": 0.11670766770839691,
|
||
|
|
"epoch": 0.25319479278633705,
|
||
|
|
"grad_norm": 34.0,
|
||
|
|
"learning_rate": 4.833951066243004e-07,
|
||
|
|
"logits/chosen": -2.0384128093719482,
|
||
|
|
"logits/rejected": -2.0463626384735107,
|
||
|
|
"logps/chosen": -0.29939448833465576,
|
||
|
|
"logps/rejected": -0.30272746086120605,
|
||
|
|
"loss": 25.318052291870117,
|
||
|
|
"loss/core": 25.299692153930664,
|
||
|
|
"loss/preference_sft": 0.29939448833465576,
|
||
|
|
"loss/reference_anchor": 0.21487000584602356,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.2312783002853394,
|
||
|
|
"rewards/margins": 0.06888408958911896,
|
||
|
|
"rewards/rejected": 1.1623942852020264,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.32817989587783813,
|
||
|
|
"drift/rejected_abs": 0.21317580342292786,
|
||
|
|
"epoch": 0.2579720530275887,
|
||
|
|
"grad_norm": 57.0,
|
||
|
|
"learning_rate": 4.813851260043915e-07,
|
||
|
|
"logits/chosen": -1.9417810440063477,
|
||
|
|
"logits/rejected": -2.031541347503662,
|
||
|
|
"logps/chosen": -0.28844502568244934,
|
||
|
|
"logps/rejected": -0.29021137952804565,
|
||
|
|
"loss": 24.536828994750977,
|
||
|
|
"loss/core": 24.490381240844727,
|
||
|
|
"loss/preference_sft": 0.28844502568244934,
|
||
|
|
"loss/reference_anchor": 0.5904703140258789,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.2817986011505127,
|
||
|
|
"rewards/margins": 1.1503736972808838,
|
||
|
|
"rewards/rejected": 2.13142466545105,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23207350075244904,
|
||
|
|
"drift/rejected_abs": 0.12252874672412872,
|
||
|
|
"epoch": 0.2627493132688403,
|
||
|
|
"grad_norm": 572.0,
|
||
|
|
"learning_rate": 4.79265018596281e-07,
|
||
|
|
"logits/chosen": -2.0927796363830566,
|
||
|
|
"logits/rejected": -2.0458498001098633,
|
||
|
|
"logps/chosen": -0.29707807302474976,
|
||
|
|
"logps/rejected": -0.3014887869358063,
|
||
|
|
"loss": 23.969472885131836,
|
||
|
|
"loss/core": 23.95018768310547,
|
||
|
|
"loss/preference_sft": 0.29707807302474976,
|
||
|
|
"loss/reference_anchor": 0.22740456461906433,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.3190271854400635,
|
||
|
|
"rewards/margins": 1.1700794696807861,
|
||
|
|
"rewards/rejected": 1.1489474773406982,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2051713466644287,
|
||
|
|
"drift/rejected_abs": 0.08534234762191772,
|
||
|
|
"epoch": 0.26752657351009196,
|
||
|
|
"grad_norm": 56.75,
|
||
|
|
"learning_rate": 4.770357934562704e-07,
|
||
|
|
"logits/chosen": -2.063467502593994,
|
||
|
|
"logits/rejected": -1.9795265197753906,
|
||
|
|
"logps/chosen": -0.2879626750946045,
|
||
|
|
"logps/rejected": -0.3061186373233795,
|
||
|
|
"loss": 23.862253189086914,
|
||
|
|
"loss/core": 23.851999282836914,
|
||
|
|
"loss/preference_sft": 0.2879626750946045,
|
||
|
|
"loss/reference_anchor": 0.10792496055364609,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0476605892181396,
|
||
|
|
"rewards/margins": 1.2737023830413818,
|
||
|
|
"rewards/rejected": 0.7739582061767578,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2606334090232849,
|
||
|
|
"drift/rejected_abs": 0.16303770244121552,
|
||
|
|
"epoch": 0.2723038337513436,
|
||
|
|
"grad_norm": 168.0,
|
||
|
|
"learning_rate": 4.746985115747917e-07,
|
||
|
|
"logits/chosen": -2.0845389366149902,
|
||
|
|
"logits/rejected": -2.0801751613616943,
|
||
|
|
"logps/chosen": -0.29716962575912476,
|
||
|
|
"logps/rejected": -0.29298409819602966,
|
||
|
|
"loss": 24.26735496520996,
|
||
|
|
"loss/core": 24.245363235473633,
|
||
|
|
"loss/preference_sft": 0.29716962575912476,
|
||
|
|
"loss/reference_anchor": 0.26353123784065247,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.605191707611084,
|
||
|
|
"rewards/margins": 0.9785102605819702,
|
||
|
|
"rewards/rejected": 1.6266816854476929,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14788036048412323,
|
||
|
|
"drift/rejected_abs": 0.14346002042293549,
|
||
|
|
"epoch": 0.27708109399259523,
|
||
|
|
"grad_norm": 1488.0,
|
||
|
|
"learning_rate": 4.722542853714341e-07,
|
||
|
|
"logits/chosen": -2.1646649837493896,
|
||
|
|
"logits/rejected": -2.176093578338623,
|
||
|
|
"logps/chosen": -0.2869679927825928,
|
||
|
|
"logps/rejected": -0.28978294134140015,
|
||
|
|
"loss": 25.25179672241211,
|
||
|
|
"loss/core": 25.234987258911133,
|
||
|
|
"loss/preference_sft": 0.2869679927825928,
|
||
|
|
"loss/reference_anchor": 0.19546569883823395,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.4774421453475952,
|
||
|
|
"rewards/margins": 0.04469200596213341,
|
||
|
|
"rewards/rejected": 1.4327499866485596,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1308569461107254,
|
||
|
|
"drift/rejected_abs": 0.0676531195640564,
|
||
|
|
"epoch": 0.28185835423384686,
|
||
|
|
"grad_norm": 64.5,
|
||
|
|
"learning_rate": 4.697042781654913e-07,
|
||
|
|
"logits/chosen": -2.565634250640869,
|
||
|
|
"logits/rejected": -2.5284571647644043,
|
||
|
|
"logps/chosen": -0.2447257786989212,
|
||
|
|
"logps/rejected": -0.2566220164299011,
|
||
|
|
"loss": 24.383955001831055,
|
||
|
|
"loss/core": 24.379867553710938,
|
||
|
|
"loss/preference_sft": 0.2447257786989212,
|
||
|
|
"loss/reference_anchor": 0.030016642063856125,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.308022379875183,
|
||
|
|
"rewards/margins": 0.6341660022735596,
|
||
|
|
"rewards/rejected": 0.6738564968109131,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3740275204181671,
|
||
|
|
"drift/rejected_abs": 0.10665237903594971,
|
||
|
|
"epoch": 0.28663561447509855,
|
||
|
|
"grad_norm": 388.0,
|
||
|
|
"learning_rate": 4.670497036222856e-07,
|
||
|
|
"logits/chosen": -1.9813871383666992,
|
||
|
|
"logits/rejected": -1.9485054016113281,
|
||
|
|
"logps/chosen": -0.31088101863861084,
|
||
|
|
"logps/rejected": -0.3178117573261261,
|
||
|
|
"loss": 23.0051326751709,
|
||
|
|
"loss/core": 22.965051651000977,
|
||
|
|
"loss/preference_sft": 0.31088101863861084,
|
||
|
|
"loss/reference_anchor": 0.5033376216888428,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.739037275314331,
|
||
|
|
"rewards/margins": 2.6878647804260254,
|
||
|
|
"rewards/rejected": 1.0511722564697266,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2159603089094162,
|
||
|
|
"drift/rejected_abs": 0.0897536352276802,
|
||
|
|
"epoch": 0.2914128747163502,
|
||
|
|
"grad_norm": 76.0,
|
||
|
|
"learning_rate": 4.642918251755281e-07,
|
||
|
|
"logits/chosen": -2.125406265258789,
|
||
|
|
"logits/rejected": -2.185771942138672,
|
||
|
|
"logps/chosen": -0.26945987343788147,
|
||
|
|
"logps/rejected": -0.26080164313316345,
|
||
|
|
"loss": 23.85554313659668,
|
||
|
|
"loss/core": 23.84500503540039,
|
||
|
|
"loss/preference_sft": 0.26945987343788147,
|
||
|
|
"loss/reference_anchor": 0.11357325315475464,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.159000873565674,
|
||
|
|
"rewards/margins": 1.2621368169784546,
|
||
|
|
"rewards/rejected": 0.896864116191864,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26018840074539185,
|
||
|
|
"drift/rejected_abs": 0.2073136568069458,
|
||
|
|
"epoch": 0.2961901349576018,
|
||
|
|
"grad_norm": 976.0,
|
||
|
|
"learning_rate": 4.614319554259933e-07,
|
||
|
|
"logits/chosen": -2.12811017036438,
|
||
|
|
"logits/rejected": -2.0370516777038574,
|
||
|
|
"logps/chosen": -0.2741633355617523,
|
||
|
|
"logps/rejected": -0.2802058160305023,
|
||
|
|
"loss": 25.01598358154297,
|
||
|
|
"loss/core": 24.98000144958496,
|
||
|
|
"loss/preference_sft": 0.2741633355617523,
|
||
|
|
"loss/reference_anchor": 0.452305406332016,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.6010148525238037,
|
||
|
|
"rewards/margins": 0.5811925530433655,
|
||
|
|
"rewards/rejected": 2.019822359085083,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29169219732284546,
|
||
|
|
"drift/rejected_abs": 0.13470803201198578,
|
||
|
|
"epoch": 0.30096739519885346,
|
||
|
|
"grad_norm": 61.0,
|
||
|
|
"learning_rate": 4.58471455516792e-07,
|
||
|
|
"logits/chosen": -1.9604593515396118,
|
||
|
|
"logits/rejected": -2.0688371658325195,
|
||
|
|
"logps/chosen": -0.2688148617744446,
|
||
|
|
"logps/rejected": -0.2757693827152252,
|
||
|
|
"loss": 23.543533325195312,
|
||
|
|
"loss/core": 23.524099349975586,
|
||
|
|
"loss/preference_sft": 0.2688148617744446,
|
||
|
|
"loss/reference_anchor": 0.23222799599170685,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.9164979457855225,
|
||
|
|
"rewards/margins": 1.5716060400009155,
|
||
|
|
"rewards/rejected": 1.344891905784607,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1583484411239624,
|
||
|
|
"drift/rejected_abs": 0.12316056340932846,
|
||
|
|
"epoch": 0.3057446554401051,
|
||
|
|
"grad_norm": 51.5,
|
||
|
|
"learning_rate": 4.5541173448554095e-07,
|
||
|
|
"logits/chosen": -2.0016112327575684,
|
||
|
|
"logits/rejected": -2.0177295207977295,
|
||
|
|
"logps/chosen": -0.2901010513305664,
|
||
|
|
"logps/rejected": -0.2924564480781555,
|
||
|
|
"loss": 24.739116668701172,
|
||
|
|
"loss/core": 24.729549407958984,
|
||
|
|
"loss/preference_sft": 0.2901010513305664,
|
||
|
|
"loss/reference_anchor": 0.09852476418018341,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.5812407732009888,
|
||
|
|
"rewards/margins": 0.35098347067832947,
|
||
|
|
"rewards/rejected": 1.2302576303482056,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36936116218566895,
|
||
|
|
"drift/rejected_abs": 0.1534915715456009,
|
||
|
|
"epoch": 0.31052191568135673,
|
||
|
|
"grad_norm": 146.0,
|
||
|
|
"learning_rate": 4.5225424859373684e-07,
|
||
|
|
"logits/chosen": -2.257908344268799,
|
||
|
|
"logits/rejected": -2.2035529613494873,
|
||
|
|
"logps/chosen": -0.28698205947875977,
|
||
|
|
"logps/rejected": -0.28376373648643494,
|
||
|
|
"loss": 24.22592544555664,
|
||
|
|
"loss/core": 24.155797958374023,
|
||
|
|
"loss/preference_sft": 0.28698205947875977,
|
||
|
|
"loss/reference_anchor": 0.9063272476196289,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.6920619010925293,
|
||
|
|
"rewards/margins": 2.15855073928833,
|
||
|
|
"rewards/rejected": 1.5335115194320679,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20329172909259796,
|
||
|
|
"drift/rejected_abs": 0.11353343725204468,
|
||
|
|
"epoch": 0.31529917592260837,
|
||
|
|
"grad_norm": 40.75,
|
||
|
|
"learning_rate": 4.4900050063365547e-07,
|
||
|
|
"logits/chosen": -2.2280595302581787,
|
||
|
|
"logits/rejected": -2.217438220977783,
|
||
|
|
"logps/chosen": -0.27844610810279846,
|
||
|
|
"logps/rejected": -0.270263135433197,
|
||
|
|
"loss": 24.234060287475586,
|
||
|
|
"loss/core": 24.221086502075195,
|
||
|
|
"loss/preference_sft": 0.27844610810279846,
|
||
|
|
"loss/reference_anchor": 0.14516863226890564,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 2.0322957038879395,
|
||
|
|
"rewards/margins": 0.8984895944595337,
|
||
|
|
"rewards/rejected": 1.1338062286376953,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2127622663974762,
|
||
|
|
"drift/rejected_abs": 0.07780848443508148,
|
||
|
|
"epoch": 0.32007643616386,
|
||
|
|
"grad_norm": 165.0,
|
||
|
|
"learning_rate": 4.4565203921310344e-07,
|
||
|
|
"logits/chosen": -2.0511183738708496,
|
||
|
|
"logits/rejected": -2.0350396633148193,
|
||
|
|
"logps/chosen": -0.30186930298805237,
|
||
|
|
"logps/rejected": -0.3197532296180725,
|
||
|
|
"loss": 23.990903854370117,
|
||
|
|
"loss/core": 23.97220230102539,
|
||
|
|
"loss/preference_sft": 0.30186930298805237,
|
||
|
|
"loss/reference_anchor": 0.21921315789222717,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.124210834503174,
|
||
|
|
"rewards/margins": 1.4477331638336182,
|
||
|
|
"rewards/rejected": 0.6764776110649109,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2031564712524414,
|
||
|
|
"drift/rejected_abs": 0.087882399559021,
|
||
|
|
"epoch": 0.3248536964051117,
|
||
|
|
"grad_norm": 186.0,
|
||
|
|
"learning_rate": 4.422104580183649e-07,
|
||
|
|
"logits/chosen": -2.095062017440796,
|
||
|
|
"logits/rejected": -2.2184524536132812,
|
||
|
|
"logps/chosen": -0.28098320960998535,
|
||
|
|
"logps/rejected": -0.28097662329673767,
|
||
|
|
"loss": 24.006999969482422,
|
||
|
|
"loss/core": 23.993459701538086,
|
||
|
|
"loss/preference_sft": 0.28098320960998535,
|
||
|
|
"loss/reference_anchor": 0.15243388712406158,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.031026840209961,
|
||
|
|
"rewards/margins": 1.153816819190979,
|
||
|
|
"rewards/rejected": 0.8772099614143372,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16465213894844055,
|
||
|
|
"drift/rejected_abs": 0.0845918357372284,
|
||
|
|
"epoch": 0.3296309566463633,
|
||
|
|
"grad_norm": 98.5,
|
||
|
|
"learning_rate": 4.3867739505569303e-07,
|
||
|
|
"logits/chosen": -2.3218281269073486,
|
||
|
|
"logits/rejected": -2.3827481269836426,
|
||
|
|
"logps/chosen": -0.282081663608551,
|
||
|
|
"logps/rejected": -0.29249638319015503,
|
||
|
|
"loss": 24.309249877929688,
|
||
|
|
"loss/core": 24.29976463317871,
|
||
|
|
"loss/preference_sft": 0.282081663608551,
|
||
|
|
"loss/reference_anchor": 0.09826230257749557,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.644649863243103,
|
||
|
|
"rewards/margins": 0.799951434135437,
|
||
|
|
"rewards/rejected": 0.8446983098983765,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20000576972961426,
|
||
|
|
"drift/rejected_abs": 0.09551592916250229,
|
||
|
|
"epoch": 0.33440821688761496,
|
||
|
|
"grad_norm": 80.5,
|
||
|
|
"learning_rate": 4.35054531871708e-07,
|
||
|
|
"logits/chosen": -2.0598835945129395,
|
||
|
|
"logits/rejected": -2.03928804397583,
|
||
|
|
"logps/chosen": -0.26030445098876953,
|
||
|
|
"logps/rejected": -0.2558044195175171,
|
||
|
|
"loss": 24.042661666870117,
|
||
|
|
"loss/core": 24.03305435180664,
|
||
|
|
"loss/preference_sft": 0.26030445098876953,
|
||
|
|
"loss/reference_anchor": 0.1020883321762085,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.9989124536514282,
|
||
|
|
"rewards/margins": 1.0452065467834473,
|
||
|
|
"rewards/rejected": 0.953705906867981,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18520495295524597,
|
||
|
|
"drift/rejected_abs": 0.1412447839975357,
|
||
|
|
"epoch": 0.3391854771288666,
|
||
|
|
"grad_norm": 75.5,
|
||
|
|
"learning_rate": 4.3134359275307185e-07,
|
||
|
|
"logits/chosen": -2.106975555419922,
|
||
|
|
"logits/rejected": -2.2341411113739014,
|
||
|
|
"logps/chosen": -0.28064435720443726,
|
||
|
|
"logps/rejected": -0.2879922389984131,
|
||
|
|
"loss": 24.90737533569336,
|
||
|
|
"loss/core": 24.887197494506836,
|
||
|
|
"loss/preference_sft": 0.28064435720443726,
|
||
|
|
"loss/reference_anchor": 0.2410183846950531,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.851447343826294,
|
||
|
|
"rewards/margins": 0.45425519347190857,
|
||
|
|
"rewards/rejected": 1.3971920013427734,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26272064447402954,
|
||
|
|
"drift/rejected_abs": 0.1274389922618866,
|
||
|
|
"epoch": 0.34396273737011823,
|
||
|
|
"grad_norm": 472.0,
|
||
|
|
"learning_rate": 4.2754634390582133e-07,
|
||
|
|
"logits/chosen": -2.1305363178253174,
|
||
|
|
"logits/rejected": -2.192857265472412,
|
||
|
|
"logps/chosen": -0.28095608949661255,
|
||
|
|
"logps/rejected": -0.28217315673828125,
|
||
|
|
"loss": 24.006431579589844,
|
||
|
|
"loss/core": 23.981016159057617,
|
||
|
|
"loss/preference_sft": 0.28095608949661255,
|
||
|
|
"loss/reference_anchor": 0.31079161167144775,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.62534761428833,
|
||
|
|
"rewards/margins": 1.353183627128601,
|
||
|
|
"rewards/rejected": 1.272163987159729,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2852028012275696,
|
||
|
|
"drift/rejected_abs": 0.08169830590486526,
|
||
|
|
"epoch": 0.34873999761136987,
|
||
|
|
"grad_norm": 205.0,
|
||
|
|
"learning_rate": 4.236645926147493e-07,
|
||
|
|
"logits/chosen": -2.069857120513916,
|
||
|
|
"logits/rejected": -2.1774981021881104,
|
||
|
|
"logps/chosen": -0.287311851978302,
|
||
|
|
"logps/rejected": -0.2833508551120758,
|
||
|
|
"loss": 23.85344123840332,
|
||
|
|
"loss/core": 23.815753936767578,
|
||
|
|
"loss/preference_sft": 0.287311851978302,
|
||
|
|
"loss/reference_anchor": 0.4737482964992523,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.851012945175171,
|
||
|
|
"rewards/margins": 2.036228895187378,
|
||
|
|
"rewards/rejected": 0.8147839307785034,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1644589602947235,
|
||
|
|
"drift/rejected_abs": 0.10670790821313858,
|
||
|
|
"epoch": 0.3535172578526215,
|
||
|
|
"grad_norm": 188.0,
|
||
|
|
"learning_rate": 4.1970018638323547e-07,
|
||
|
|
"logits/chosen": -2.2686495780944824,
|
||
|
|
"logits/rejected": -2.3140177726745605,
|
||
|
|
"logps/chosen": -0.2849913239479065,
|
||
|
|
"logps/rejected": -0.2848779857158661,
|
||
|
|
"loss": 24.466651916503906,
|
||
|
|
"loss/core": 24.457897186279297,
|
||
|
|
"loss/preference_sft": 0.2849913239479065,
|
||
|
|
"loss/reference_anchor": 0.08822296559810638,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6427894830703735,
|
||
|
|
"rewards/margins": 0.5768302083015442,
|
||
|
|
"rewards/rejected": 1.0659592151641846,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3016364276409149,
|
||
|
|
"drift/rejected_abs": 0.22407476603984833,
|
||
|
|
"epoch": 0.35829451809387314,
|
||
|
|
"grad_norm": 70.5,
|
||
|
|
"learning_rate": 4.1565501205393436e-07,
|
||
|
|
"logits/chosen": -2.0465548038482666,
|
||
|
|
"logits/rejected": -2.024115562438965,
|
||
|
|
"logps/chosen": -0.25850996375083923,
|
||
|
|
"logps/rejected": -0.25814658403396606,
|
||
|
|
"loss": 24.701723098754883,
|
||
|
|
"loss/core": 24.66933822631836,
|
||
|
|
"loss/preference_sft": 0.25850996375083923,
|
||
|
|
"loss/reference_anchor": 0.4059620499610901,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.0161895751953125,
|
||
|
|
"rewards/margins": 0.7962467670440674,
|
||
|
|
"rewards/rejected": 2.2199432849884033,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20642168819904327,
|
||
|
|
"drift/rejected_abs": 0.16034074127674103,
|
||
|
|
"epoch": 0.36307177833512483,
|
||
|
|
"grad_norm": 106.0,
|
||
|
|
"learning_rate": 4.1153099491074093e-07,
|
||
|
|
"logits/chosen": -2.122387409210205,
|
||
|
|
"logits/rejected": -2.131068468093872,
|
||
|
|
"logps/chosen": -0.27889400720596313,
|
||
|
|
"logps/rejected": -0.2892562448978424,
|
||
|
|
"loss": 25.11880874633789,
|
||
|
|
"loss/core": 25.087757110595703,
|
||
|
|
"loss/preference_sft": 0.27889400720596313,
|
||
|
|
"loss/reference_anchor": 0.38615480065345764,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.06227445602417,
|
||
|
|
"rewards/margins": 0.45964282751083374,
|
||
|
|
"rewards/rejected": 1.6026318073272705,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21806609630584717,
|
||
|
|
"drift/rejected_abs": 0.15287581086158752,
|
||
|
|
"epoch": 0.36784903857637646,
|
||
|
|
"grad_norm": 131.0,
|
||
|
|
"learning_rate": 4.0733009776245937e-07,
|
||
|
|
"logits/chosen": -2.115004062652588,
|
||
|
|
"logits/rejected": -2.1338284015655518,
|
||
|
|
"logps/chosen": -0.2700949013233185,
|
||
|
|
"logps/rejected": -0.27447065711021423,
|
||
|
|
"loss": 24.529088973999023,
|
||
|
|
"loss/core": 24.512638092041016,
|
||
|
|
"loss/preference_sft": 0.2700949013233185,
|
||
|
|
"loss/reference_anchor": 0.19232222437858582,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.179309129714966,
|
||
|
|
"rewards/margins": 0.6530361175537109,
|
||
|
|
"rewards/rejected": 1.5262731313705444,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17654861509799957,
|
||
|
|
"drift/rejected_abs": 0.09032897651195526,
|
||
|
|
"epoch": 0.3726262988176281,
|
||
|
|
"grad_norm": 42.75,
|
||
|
|
"learning_rate": 4.0305432000861226e-07,
|
||
|
|
"logits/chosen": -1.8726367950439453,
|
||
|
|
"logits/rejected": -1.9506858587265015,
|
||
|
|
"logps/chosen": -0.3072229325771332,
|
||
|
|
"logps/rejected": -0.3017742931842804,
|
||
|
|
"loss": 24.248693466186523,
|
||
|
|
"loss/core": 24.23879051208496,
|
||
|
|
"loss/preference_sft": 0.3072229325771332,
|
||
|
|
"loss/reference_anchor": 0.10132338106632233,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.761505126953125,
|
||
|
|
"rewards/margins": 0.8584615588188171,
|
||
|
|
"rewards/rejected": 0.9030437469482422,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1685396432876587,
|
||
|
|
"drift/rejected_abs": 0.08101732283830643,
|
||
|
|
"epoch": 0.37740355905887973,
|
||
|
|
"grad_norm": 101.0,
|
||
|
|
"learning_rate": 3.9870569668783533e-07,
|
||
|
|
"logits/chosen": -2.223672389984131,
|
||
|
|
"logits/rejected": -2.3257579803466797,
|
||
|
|
"logps/chosen": -0.29469752311706543,
|
||
|
|
"logps/rejected": -0.28933185338974,
|
||
|
|
"loss": 24.182411193847656,
|
||
|
|
"loss/core": 24.174169540405273,
|
||
|
|
"loss/preference_sft": 0.29469752311706543,
|
||
|
|
"loss/reference_anchor": 0.08043049275875092,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 1.6851272583007812,
|
||
|
|
"rewards/margins": 0.8753724098205566,
|
||
|
|
"rewards/rejected": 0.8097550272941589,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29273721575737,
|
||
|
|
"drift/rejected_abs": 0.0846409797668457,
|
||
|
|
"epoch": 0.38218081930013137,
|
||
|
|
"grad_norm": 85.0,
|
||
|
|
"learning_rate": 3.942862975093084e-07,
|
||
|
|
"logits/chosen": -2.1020679473876953,
|
||
|
|
"logits/rejected": -2.1372578144073486,
|
||
|
|
"logps/chosen": -0.29274052381515503,
|
||
|
|
"logps/rejected": -0.2928157448768616,
|
||
|
|
"loss": 23.699491500854492,
|
||
|
|
"loss/core": 23.65845489501953,
|
||
|
|
"loss/preference_sft": 0.29274052381515503,
|
||
|
|
"loss/reference_anchor": 0.5178688168525696,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.925940990447998,
|
||
|
|
"rewards/margins": 2.0813891887664795,
|
||
|
|
"rewards/rejected": 0.8445518612861633,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2351594716310501,
|
||
|
|
"drift/rejected_abs": 0.12229617685079575,
|
||
|
|
"epoch": 0.386958079541383,
|
||
|
|
"grad_norm": 65.0,
|
||
|
|
"learning_rate": 3.8979822586768666e-07,
|
||
|
|
"logits/chosen": -2.135050058364868,
|
||
|
|
"logits/rejected": -2.189692735671997,
|
||
|
|
"logps/chosen": -0.3113452196121216,
|
||
|
|
"logps/rejected": -0.3050457835197449,
|
||
|
|
"loss": 24.190441131591797,
|
||
|
|
"loss/core": 24.1619873046875,
|
||
|
|
"loss/preference_sft": 0.3113452196121216,
|
||
|
|
"loss/reference_anchor": 0.3482899069786072,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.3515946865081787,
|
||
|
|
"rewards/margins": 1.1289160251617432,
|
||
|
|
"rewards/rejected": 1.2226788997650146,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20916077494621277,
|
||
|
|
"drift/rejected_abs": 0.11924557387828827,
|
||
|
|
"epoch": 0.39173533978263464,
|
||
|
|
"grad_norm": 86.0,
|
||
|
|
"learning_rate": 3.8524361784199847e-07,
|
||
|
|
"logits/chosen": -2.251237630844116,
|
||
|
|
"logits/rejected": -2.2815263271331787,
|
||
|
|
"logps/chosen": -0.27680549025535583,
|
||
|
|
"logps/rejected": -0.2878721058368683,
|
||
|
|
"loss": 24.154521942138672,
|
||
|
|
"loss/core": 24.142080307006836,
|
||
|
|
"loss/preference_sft": 0.27680549025535583,
|
||
|
|
"loss/reference_anchor": 0.1381910741329193,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.090045690536499,
|
||
|
|
"rewards/margins": 0.8996987342834473,
|
||
|
|
"rewards/rejected": 1.1903469562530518,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.09965378046035767,
|
||
|
|
"drift/rejected_abs": 0.037877295166254044,
|
||
|
|
"epoch": 0.3965126000238863,
|
||
|
|
"grad_norm": 77.5,
|
||
|
|
"learning_rate": 3.806246411789872e-07,
|
||
|
|
"logits/chosen": -2.1194329261779785,
|
||
|
|
"logits/rejected": -2.1981375217437744,
|
||
|
|
"logps/chosen": -0.33038097620010376,
|
||
|
|
"logps/rejected": -0.32079142332077026,
|
||
|
|
"loss": 24.420391082763672,
|
||
|
|
"loss/core": 24.41567039489746,
|
||
|
|
"loss/preference_sft": 0.33038097620010376,
|
||
|
|
"loss/reference_anchor": 0.02990715578198433,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 0.9942773580551147,
|
||
|
|
"rewards/margins": 0.616710364818573,
|
||
|
|
"rewards/rejected": 0.37756696343421936,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2046138346195221,
|
||
|
|
"drift/rejected_abs": 0.07252134382724762,
|
||
|
|
"epoch": 0.40128986026513797,
|
||
|
|
"grad_norm": 129.0,
|
||
|
|
"learning_rate": 3.7594349426138154e-07,
|
||
|
|
"logits/chosen": -2.350454807281494,
|
||
|
|
"logits/rejected": -2.316537618637085,
|
||
|
|
"logps/chosen": -0.2832780182361603,
|
||
|
|
"logps/rejected": -0.29880502820014954,
|
||
|
|
"loss": 23.930326461791992,
|
||
|
|
"loss/core": 23.915685653686523,
|
||
|
|
"loss/preference_sft": 0.2832780182361603,
|
||
|
|
"loss/reference_anchor": 0.1668982058763504,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.046138286590576,
|
||
|
|
"rewards/margins": 1.3250911235809326,
|
||
|
|
"rewards/rejected": 0.721047043800354,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2236243188381195,
|
||
|
|
"drift/rejected_abs": 0.1645563691854477,
|
||
|
|
"epoch": 0.4060671205063896,
|
||
|
|
"grad_norm": 486.0,
|
||
|
|
"learning_rate": 3.712024050615843e-07,
|
||
|
|
"logits/chosen": -2.0146186351776123,
|
||
|
|
"logits/rejected": -2.0732080936431885,
|
||
|
|
"logps/chosen": -0.2916070222854614,
|
||
|
|
"logps/rejected": -0.2652800679206848,
|
||
|
|
"loss": 25.089502334594727,
|
||
|
|
"loss/core": 25.058069229125977,
|
||
|
|
"loss/preference_sft": 0.2916070222854614,
|
||
|
|
"loss/reference_anchor": 0.3899894654750824,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.235166549682617,
|
||
|
|
"rewards/margins": 0.5908761620521545,
|
||
|
|
"rewards/rejected": 1.6442902088165283,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.44021230936050415,
|
||
|
|
"drift/rejected_abs": 0.1417957842350006,
|
||
|
|
"epoch": 0.41084438074764124,
|
||
|
|
"grad_norm": 274.0,
|
||
|
|
"learning_rate": 3.664036300812778e-07,
|
||
|
|
"logits/chosen": -2.0683376789093018,
|
||
|
|
"logits/rejected": -2.091358184814453,
|
||
|
|
"logps/chosen": -0.27976277470588684,
|
||
|
|
"logps/rejected": -0.2667387127876282,
|
||
|
|
"loss": 23.26540184020996,
|
||
|
|
"loss/core": 23.194873809814453,
|
||
|
|
"loss/preference_sft": 0.27976277470588684,
|
||
|
|
"loss/reference_anchor": 0.9124003648757935,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.400694847106934,
|
||
|
|
"rewards/margins": 2.9839932918548584,
|
||
|
|
"rewards/rejected": 1.4167020320892334,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16288933157920837,
|
||
|
|
"drift/rejected_abs": 0.10126354545354843,
|
||
|
|
"epoch": 0.41562164098889287,
|
||
|
|
"grad_norm": 99.5,
|
||
|
|
"learning_rate": 3.615494532774522e-07,
|
||
|
|
"logits/chosen": -2.3494062423706055,
|
||
|
|
"logits/rejected": -2.3344578742980957,
|
||
|
|
"logps/chosen": -0.2779366374015808,
|
||
|
|
"logps/rejected": -0.29263314604759216,
|
||
|
|
"loss": 24.44803810119629,
|
||
|
|
"loss/core": 24.437061309814453,
|
||
|
|
"loss/preference_sft": 0.2779366374015808,
|
||
|
|
"loss/reference_anchor": 0.11851142346858978,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.628753900527954,
|
||
|
|
"rewards/margins": 0.6175113916397095,
|
||
|
|
"rewards/rejected": 1.0112426280975342,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.30069416761398315,
|
||
|
|
"drift/rejected_abs": 0.13946831226348877,
|
||
|
|
"epoch": 0.4203989012301445,
|
||
|
|
"grad_norm": 668.0,
|
||
|
|
"learning_rate": 3.5664218497536456e-07,
|
||
|
|
"logits/chosen": -2.05379581451416,
|
||
|
|
"logits/rejected": -2.1259214878082275,
|
||
|
|
"logps/chosen": -0.303558886051178,
|
||
|
|
"logps/rejected": -0.3001348078250885,
|
||
|
|
"loss": 24.09839630126953,
|
||
|
|
"loss/core": 24.058101654052734,
|
||
|
|
"loss/preference_sft": 0.303558886051178,
|
||
|
|
"loss/reference_anchor": 0.5069498419761658,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.003722667694092,
|
||
|
|
"rewards/margins": 1.6185331344604492,
|
||
|
|
"rewards/rejected": 1.3851896524429321,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29670435190200806,
|
||
|
|
"drift/rejected_abs": 0.16009071469306946,
|
||
|
|
"epoch": 0.42517616147139614,
|
||
|
|
"grad_norm": 130.0,
|
||
|
|
"learning_rate": 3.516841607689501e-07,
|
||
|
|
"logits/chosen": -1.9354779720306396,
|
||
|
|
"logits/rejected": -1.9123506546020508,
|
||
|
|
"logps/chosen": -0.29504531621932983,
|
||
|
|
"logps/rejected": -0.2956696152687073,
|
||
|
|
"loss": 23.822757720947266,
|
||
|
|
"loss/core": 23.801040649414062,
|
||
|
|
"loss/preference_sft": 0.29504531621932983,
|
||
|
|
"loss/reference_anchor": 0.2600812315940857,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.9661905765533447,
|
||
|
|
"rewards/margins": 1.388580560684204,
|
||
|
|
"rewards/rejected": 1.577609896659851,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.345471054315567,
|
||
|
|
"drift/rejected_abs": 0.161865234375,
|
||
|
|
"epoch": 0.4299534217126478,
|
||
|
|
"grad_norm": 768.0,
|
||
|
|
"learning_rate": 3.4667774040920515e-07,
|
||
|
|
"logits/chosen": -1.9765275716781616,
|
||
|
|
"logits/rejected": -1.9926903247833252,
|
||
|
|
"logps/chosen": -0.29864731431007385,
|
||
|
|
"logps/rejected": -0.2888573110103607,
|
||
|
|
"loss": 23.699243545532227,
|
||
|
|
"loss/core": 23.66305160522461,
|
||
|
|
"loss/preference_sft": 0.29864731431007385,
|
||
|
|
"loss/reference_anchor": 0.4526439309120178,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.4545040130615234,
|
||
|
|
"rewards/margins": 1.8422412872314453,
|
||
|
|
"rewards/rejected": 1.6122627258300781,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.34665170311927795,
|
||
|
|
"drift/rejected_abs": 0.14890094101428986,
|
||
|
|
"epoch": 0.4347306819538994,
|
||
|
|
"grad_norm": 246.0,
|
||
|
|
"learning_rate": 3.416253066810743e-07,
|
||
|
|
"logits/chosen": -1.936833143234253,
|
||
|
|
"logits/rejected": -2.127368927001953,
|
||
|
|
"logps/chosen": -0.32262879610061646,
|
||
|
|
"logps/rejected": -0.31309956312179565,
|
||
|
|
"loss": 23.719173431396484,
|
||
|
|
"loss/core": 23.66214370727539,
|
||
|
|
"loss/preference_sft": 0.32262879610061646,
|
||
|
|
"loss/reference_anchor": 0.728130042552948,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.4665169715881348,
|
||
|
|
"rewards/margins": 1.979360580444336,
|
||
|
|
"rewards/rejected": 1.4871563911437988,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36950841546058655,
|
||
|
|
"drift/rejected_abs": 0.18701323866844177,
|
||
|
|
"epoch": 0.4395079421951511,
|
||
|
|
"grad_norm": 141.0,
|
||
|
|
"learning_rate": 3.3652926426937325e-07,
|
||
|
|
"logits/chosen": -2.0627896785736084,
|
||
|
|
"logits/rejected": -2.0499472618103027,
|
||
|
|
"logps/chosen": -0.2930159270763397,
|
||
|
|
"logps/rejected": -0.3065492510795593,
|
||
|
|
"loss": 23.571096420288086,
|
||
|
|
"loss/core": 23.52437973022461,
|
||
|
|
"loss/preference_sft": 0.2930159270763397,
|
||
|
|
"loss/reference_anchor": 0.5935932397842407,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.694608688354492,
|
||
|
|
"rewards/margins": 1.8250821828842163,
|
||
|
|
"rewards/rejected": 1.8695262670516968,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23000410199165344,
|
||
|
|
"drift/rejected_abs": 0.09393614530563354,
|
||
|
|
"epoch": 0.44428520243640274,
|
||
|
|
"grad_norm": 60.25,
|
||
|
|
"learning_rate": 3.3139203861428914e-07,
|
||
|
|
"logits/chosen": -1.8922626972198486,
|
||
|
|
"logits/rejected": -1.9913305044174194,
|
||
|
|
"logps/chosen": -0.29730328917503357,
|
||
|
|
"logps/rejected": -0.29092690348625183,
|
||
|
|
"loss": 23.748533248901367,
|
||
|
|
"loss/core": 23.73752212524414,
|
||
|
|
"loss/preference_sft": 0.29730328917503357,
|
||
|
|
"loss/reference_anchor": 0.11708039045333862,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.2995853424072266,
|
||
|
|
"rewards/margins": 1.3625905513763428,
|
||
|
|
"rewards/rejected": 0.936994731426239,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.30310288071632385,
|
||
|
|
"drift/rejected_abs": 0.12414111942052841,
|
||
|
|
"epoch": 0.4490624626776544,
|
||
|
|
"grad_norm": 208.0,
|
||
|
|
"learning_rate": 3.262160747570027e-07,
|
||
|
|
"logits/chosen": -2.1301331520080566,
|
||
|
|
"logits/rejected": -2.198742628097534,
|
||
|
|
"logps/chosen": -0.27701377868652344,
|
||
|
|
"logps/rejected": -0.26015716791152954,
|
||
|
|
"loss": 23.711040496826172,
|
||
|
|
"loss/core": 23.664438247680664,
|
||
|
|
"loss/preference_sft": 0.27701377868652344,
|
||
|
|
"loss/reference_anchor": 0.5936552286148071,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.030449151992798,
|
||
|
|
"rewards/margins": 1.86244797706604,
|
||
|
|
"rewards/rejected": 1.168001413345337,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18387170135974884,
|
||
|
|
"drift/rejected_abs": 0.15331368148326874,
|
||
|
|
"epoch": 0.453839722918906,
|
||
|
|
"grad_norm": 332.0,
|
||
|
|
"learning_rate": 3.210038361759807e-07,
|
||
|
|
"logits/chosen": -2.095447301864624,
|
||
|
|
"logits/rejected": -2.033949613571167,
|
||
|
|
"logps/chosen": -0.29470914602279663,
|
||
|
|
"logps/rejected": -0.3144375681877136,
|
||
|
|
"loss": 25.121068954467773,
|
||
|
|
"loss/core": 25.099435806274414,
|
||
|
|
"loss/preference_sft": 0.29470914602279663,
|
||
|
|
"loss/reference_anchor": 0.2589879631996155,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.838440179824829,
|
||
|
|
"rewards/margins": 0.307887464761734,
|
||
|
|
"rewards/rejected": 1.530552625656128,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2358885258436203,
|
||
|
|
"drift/rejected_abs": 0.10619956254959106,
|
||
|
|
"epoch": 0.45861698316015764,
|
||
|
|
"grad_norm": 70.0,
|
||
|
|
"learning_rate": 3.1575780361449364e-07,
|
||
|
|
"logits/chosen": -2.264855146408081,
|
||
|
|
"logits/rejected": -2.3214104175567627,
|
||
|
|
"logps/chosen": -0.2755188047885895,
|
||
|
|
"logps/rejected": -0.27138248085975647,
|
||
|
|
"loss": 24.032466888427734,
|
||
|
|
"loss/core": 24.010181427001953,
|
||
|
|
"loss/preference_sft": 0.2755188047885895,
|
||
|
|
"loss/reference_anchor": 0.2696211636066437,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.3573050498962402,
|
||
|
|
"rewards/margins": 1.3217544555664062,
|
||
|
|
"rewards/rejected": 1.0355504751205444,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1745290607213974,
|
||
|
|
"drift/rejected_abs": 0.08883767575025558,
|
||
|
|
"epoch": 0.4633942434014093,
|
||
|
|
"grad_norm": 74.0,
|
||
|
|
"learning_rate": 3.104804738999169e-07,
|
||
|
|
"logits/chosen": -2.205657482147217,
|
||
|
|
"logits/rejected": -2.2127983570098877,
|
||
|
|
"logps/chosen": -0.29136472940444946,
|
||
|
|
"logps/rejected": -0.29586881399154663,
|
||
|
|
"loss": 24.19198226928711,
|
||
|
|
"loss/core": 24.183902740478516,
|
||
|
|
"loss/preference_sft": 0.29136472940444946,
|
||
|
|
"loss/reference_anchor": 0.07860977947711945,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.74505615234375,
|
||
|
|
"rewards/margins": 0.8590679168701172,
|
||
|
|
"rewards/rejected": 0.8859883546829224,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20774760842323303,
|
||
|
|
"drift/rejected_abs": 0.10662759840488434,
|
||
|
|
"epoch": 0.4681715036426609,
|
||
|
|
"grad_norm": 99.5,
|
||
|
|
"learning_rate": 3.0517435875537536e-07,
|
||
|
|
"logits/chosen": -2.1565749645233154,
|
||
|
|
"logits/rejected": -2.198429822921753,
|
||
|
|
"logps/chosen": -0.30721935629844666,
|
||
|
|
"logps/rejected": -0.29325515031814575,
|
||
|
|
"loss": 24.078283309936523,
|
||
|
|
"loss/core": 24.06472396850586,
|
||
|
|
"loss/preference_sft": 0.30721935629844666,
|
||
|
|
"loss/reference_anchor": 0.15004561841487885,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.0768425464630127,
|
||
|
|
"rewards/margins": 1.012283444404602,
|
||
|
|
"rewards/rejected": 1.0645591020584106,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2635328471660614,
|
||
|
|
"drift/rejected_abs": 0.08991514146327972,
|
||
|
|
"epoch": 0.47294876388391255,
|
||
|
|
"grad_norm": 133.0,
|
||
|
|
"learning_rate": 2.998419836042993e-07,
|
||
|
|
"logits/chosen": -2.2264866828918457,
|
||
|
|
"logits/rejected": -2.3216514587402344,
|
||
|
|
"logps/chosen": -0.29546287655830383,
|
||
|
|
"logps/rejected": -0.2902601361274719,
|
||
|
|
"loss": 23.78059959411621,
|
||
|
|
"loss/core": 23.750764846801758,
|
||
|
|
"loss/preference_sft": 0.29546287655830383,
|
||
|
|
"loss/reference_anchor": 0.36827731132507324,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.6349475383758545,
|
||
|
|
"rewards/margins": 1.7373361587524414,
|
||
|
|
"rewards/rejected": 0.897611141204834,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4189828336238861,
|
||
|
|
"drift/rejected_abs": 0.20560988783836365,
|
||
|
|
"epoch": 0.47772602412516424,
|
||
|
|
"grad_norm": 284.0,
|
||
|
|
"learning_rate": 2.9448588636846043e-07,
|
||
|
|
"logits/chosen": -2.023249387741089,
|
||
|
|
"logits/rejected": -1.972783088684082,
|
||
|
|
"logps/chosen": -0.2784698009490967,
|
||
|
|
"logps/rejected": -0.2997981607913971,
|
||
|
|
"loss": 24.244253158569336,
|
||
|
|
"loss/core": 24.155033111572266,
|
||
|
|
"loss/preference_sft": 0.2784698009490967,
|
||
|
|
"loss/reference_anchor": 1.161747694015503,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 4.189828395843506,
|
||
|
|
"rewards/margins": 2.1354339122772217,
|
||
|
|
"rewards/rejected": 2.054394245147705,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3628165125846863,
|
||
|
|
"drift/rejected_abs": 0.19208696484565735,
|
||
|
|
"epoch": 0.4825032843664159,
|
||
|
|
"grad_norm": 76.0,
|
||
|
|
"learning_rate": 2.8910861626005773e-07,
|
||
|
|
"logits/chosen": -2.1512835025787354,
|
||
|
|
"logits/rejected": -2.1834206581115723,
|
||
|
|
"logps/chosen": -0.2839333713054657,
|
||
|
|
"logps/rejected": -0.278773695230484,
|
||
|
|
"loss": 24.71078109741211,
|
||
|
|
"loss/core": 24.65015983581543,
|
||
|
|
"loss/preference_sft": 0.2839333713054657,
|
||
|
|
"loss/reference_anchor": 0.7799011468887329,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 3.6272101402282715,
|
||
|
|
"rewards/margins": 1.7839975357055664,
|
||
|
|
"rewards/rejected": 1.8432128429412842,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2025258094072342,
|
||
|
|
"drift/rejected_abs": 0.08022260665893555,
|
||
|
|
"epoch": 0.4872805446076675,
|
||
|
|
"grad_norm": 27.5,
|
||
|
|
"learning_rate": 2.8371273256843074e-07,
|
||
|
|
"logits/chosen": -2.1484246253967285,
|
||
|
|
"logits/rejected": -2.282355546951294,
|
||
|
|
"logps/chosen": -0.3147265315055847,
|
||
|
|
"logps/rejected": -0.2912803888320923,
|
||
|
|
"loss": 24.102039337158203,
|
||
|
|
"loss/core": 24.08344078063965,
|
||
|
|
"loss/preference_sft": 0.3147265315055847,
|
||
|
|
"loss/reference_anchor": 0.21651633083820343,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.022645950317383,
|
||
|
|
"rewards/margins": 1.2219388484954834,
|
||
|
|
"rewards/rejected": 0.8007070422172546,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2555674910545349,
|
||
|
|
"drift/rejected_abs": 0.16905184090137482,
|
||
|
|
"epoch": 0.49205780484891914,
|
||
|
|
"grad_norm": 129.0,
|
||
|
|
"learning_rate": 2.783008034419767e-07,
|
||
|
|
"logits/chosen": -2.032346487045288,
|
||
|
|
"logits/rejected": -2.036229372024536,
|
||
|
|
"logps/chosen": -0.2818189561367035,
|
||
|
|
"logps/rejected": -0.30472418665885925,
|
||
|
|
"loss": 24.25507164001465,
|
||
|
|
"loss/core": 24.236268997192383,
|
||
|
|
"loss/preference_sft": 0.2818189561367035,
|
||
|
|
"loss/reference_anchor": 0.22251644730567932,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 2.553283214569092,
|
||
|
|
"rewards/margins": 0.8822887539863586,
|
||
|
|
"rewards/rejected": 1.670994758605957,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.30994483828544617,
|
||
|
|
"drift/rejected_abs": 0.09894011914730072,
|
||
|
|
"epoch": 0.4968350650901708,
|
||
|
|
"grad_norm": 111.0,
|
||
|
|
"learning_rate": 2.7287540466585064e-07,
|
||
|
|
"logits/chosen": -2.059782028198242,
|
||
|
|
"logits/rejected": -2.228891372680664,
|
||
|
|
"logps/chosen": -0.3199217915534973,
|
||
|
|
"logps/rejected": -0.333048939704895,
|
||
|
|
"loss": 23.464609146118164,
|
||
|
|
"loss/core": 23.430858612060547,
|
||
|
|
"loss/preference_sft": 0.3199217915534973,
|
||
|
|
"loss/reference_anchor": 0.41807693243026733,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.0952601432800293,
|
||
|
|
"rewards/margins": 2.12144136428833,
|
||
|
|
"rewards/rejected": 0.9738188982009888,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.43741607666015625,
|
||
|
|
"drift/rejected_abs": 0.14899471402168274,
|
||
|
|
"epoch": 0.5016123253314224,
|
||
|
|
"grad_norm": 668.0,
|
||
|
|
"learning_rate": 2.674391184360313e-07,
|
||
|
|
"logits/chosen": -2.195627450942993,
|
||
|
|
"logits/rejected": -2.1632399559020996,
|
||
|
|
"logps/chosen": -0.2940196394920349,
|
||
|
|
"logps/rejected": -0.2978762984275818,
|
||
|
|
"loss": 23.785734176635742,
|
||
|
|
"loss/core": 23.704212188720703,
|
||
|
|
"loss/preference_sft": 0.2940196394920349,
|
||
|
|
"loss/reference_anchor": 1.057555079460144,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.371184349060059,
|
||
|
|
"rewards/margins": 2.8844447135925293,
|
||
|
|
"rewards/rejected": 1.4867396354675293,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3064503073692322,
|
||
|
|
"drift/rejected_abs": 0.1510452926158905,
|
||
|
|
"epoch": 0.5063895855726741,
|
||
|
|
"grad_norm": 27.0,
|
||
|
|
"learning_rate": 2.6199453213033593e-07,
|
||
|
|
"logits/chosen": -2.0389158725738525,
|
||
|
|
"logits/rejected": -2.002628803253174,
|
||
|
|
"logps/chosen": -0.28475403785705566,
|
||
|
|
"logps/rejected": -0.28787198662757874,
|
||
|
|
"loss": 23.84844970703125,
|
||
|
|
"loss/core": 23.816450119018555,
|
||
|
|
"loss/preference_sft": 0.28475403785705566,
|
||
|
|
"loss/reference_anchor": 0.3981722593307495,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.063196897506714,
|
||
|
|
"rewards/margins": 1.5666277408599854,
|
||
|
|
"rewards/rejected": 1.496569275856018,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3102155029773712,
|
||
|
|
"drift/rejected_abs": 0.10850729793310165,
|
||
|
|
"epoch": 0.5111668458139257,
|
||
|
|
"grad_norm": 58.0,
|
||
|
|
"learning_rate": 2.565442370769683e-07,
|
||
|
|
"logits/chosen": -2.1870720386505127,
|
||
|
|
"logits/rejected": -2.109102487564087,
|
||
|
|
"logps/chosen": -0.2847766578197479,
|
||
|
|
"logps/rejected": -0.27864956855773926,
|
||
|
|
"loss": 23.705360412597656,
|
||
|
|
"loss/core": 23.66876220703125,
|
||
|
|
"loss/preference_sft": 0.2847766578197479,
|
||
|
|
"loss/reference_anchor": 0.4595085680484772,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.1017348766326904,
|
||
|
|
"rewards/margins": 2.018510580062866,
|
||
|
|
"rewards/rejected": 1.0832241773605347,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15850523114204407,
|
||
|
|
"drift/rejected_abs": 0.15349014103412628,
|
||
|
|
"epoch": 0.5159441060551774,
|
||
|
|
"grad_norm": 40.5,
|
||
|
|
"learning_rate": 2.510908273211866e-07,
|
||
|
|
"logits/chosen": -2.1092724800109863,
|
||
|
|
"logits/rejected": -2.0874218940734863,
|
||
|
|
"logps/chosen": -0.27907878160476685,
|
||
|
|
"logps/rejected": -0.2851378917694092,
|
||
|
|
"loss": 25.15981674194336,
|
||
|
|
"loss/core": 25.145463943481445,
|
||
|
|
"loss/preference_sft": 0.27907878160476685,
|
||
|
|
"loss/reference_anchor": 0.163437157869339,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.5850121974945068,
|
||
|
|
"rewards/margins": 0.05249936506152153,
|
||
|
|
"rewards/rejected": 1.532512903213501,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2638896703720093,
|
||
|
|
"drift/rejected_abs": 0.09009301662445068,
|
||
|
|
"epoch": 0.520721366296429,
|
||
|
|
"grad_norm": 344.0,
|
||
|
|
"learning_rate": 2.456368983906791e-07,
|
||
|
|
"logits/chosen": -2.334498882293701,
|
||
|
|
"logits/rejected": -2.1754422187805176,
|
||
|
|
"logps/chosen": -0.28355711698532104,
|
||
|
|
"logps/rejected": -0.2911638617515564,
|
||
|
|
"loss": 23.94521141052246,
|
||
|
|
"loss/core": 23.90779685974121,
|
||
|
|
"loss/preference_sft": 0.28355711698532104,
|
||
|
|
"loss/reference_anchor": 0.47052621841430664,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.637960433959961,
|
||
|
|
"rewards/margins": 1.7409608364105225,
|
||
|
|
"rewards/rejected": 0.8969996571540833,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2771431505680084,
|
||
|
|
"drift/rejected_abs": 0.16234190762043,
|
||
|
|
"epoch": 0.5254986265376806,
|
||
|
|
"grad_norm": 94.0,
|
||
|
|
"learning_rate": 2.401850460602329e-07,
|
||
|
|
"logits/chosen": -1.901158332824707,
|
||
|
|
"logits/rejected": -1.9195445775985718,
|
||
|
|
"logps/chosen": -0.27585816383361816,
|
||
|
|
"logps/rejected": -0.29788321256637573,
|
||
|
|
"loss": 23.990930557250977,
|
||
|
|
"loss/core": 23.959190368652344,
|
||
|
|
"loss/preference_sft": 0.27585816383361816,
|
||
|
|
"loss/reference_anchor": 0.3956103026866913,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.7710883617401123,
|
||
|
|
"rewards/margins": 1.148720383644104,
|
||
|
|
"rewards/rejected": 1.6223682165145874,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.33259114623069763,
|
||
|
|
"drift/rejected_abs": 0.10077937692403793,
|
||
|
|
"epoch": 0.5302758867789323,
|
||
|
|
"grad_norm": 55.5,
|
||
|
|
"learning_rate": 2.3473786511628575e-07,
|
||
|
|
"logits/chosen": -2.095702648162842,
|
||
|
|
"logits/rejected": -2.1536788940429688,
|
||
|
|
"logps/chosen": -0.308633029460907,
|
||
|
|
"logps/rejected": -0.2737549841403961,
|
||
|
|
"loss": 23.32325553894043,
|
||
|
|
"loss/core": 23.289531707763672,
|
||
|
|
"loss/preference_sft": 0.308633029460907,
|
||
|
|
"loss/reference_anchor": 0.4187850058078766,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.3254246711730957,
|
||
|
|
"rewards/margins": 2.317995548248291,
|
||
|
|
"rewards/rejected": 1.0074293613433838,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2825181186199188,
|
||
|
|
"drift/rejected_abs": 0.15401551127433777,
|
||
|
|
"epoch": 0.5350531470201839,
|
||
|
|
"grad_norm": 135.0,
|
||
|
|
"learning_rate": 2.2929794812194894e-07,
|
||
|
|
"logits/chosen": -2.1838390827178955,
|
||
|
|
"logits/rejected": -2.171027660369873,
|
||
|
|
"logps/chosen": -0.2748723030090332,
|
||
|
|
"logps/rejected": -0.2799665927886963,
|
||
|
|
"loss": 23.88632583618164,
|
||
|
|
"loss/core": 23.858224868774414,
|
||
|
|
"loss/preference_sft": 0.2748723030090332,
|
||
|
|
"loss/reference_anchor": 0.3471968472003937,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.8238039016723633,
|
||
|
|
"rewards/margins": 1.2850908041000366,
|
||
|
|
"rewards/rejected": 1.5387130975723267,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.280038982629776,
|
||
|
|
"drift/rejected_abs": 0.16534456610679626,
|
||
|
|
"epoch": 0.5398304072614356,
|
||
|
|
"grad_norm": 140.0,
|
||
|
|
"learning_rate": 2.2386788418308665e-07,
|
||
|
|
"logits/chosen": -2.077315092086792,
|
||
|
|
"logits/rejected": -2.1030631065368652,
|
||
|
|
"logps/chosen": -0.28564533591270447,
|
||
|
|
"logps/rejected": -0.3059332072734833,
|
||
|
|
"loss": 24.126893997192383,
|
||
|
|
"loss/core": 24.075992584228516,
|
||
|
|
"loss/preference_sft": 0.28564533591270447,
|
||
|
|
"loss/reference_anchor": 0.6501015424728394,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.7995214462280273,
|
||
|
|
"rewards/margins": 1.1462886333465576,
|
||
|
|
"rewards/rejected": 1.6532329320907593,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3362275958061218,
|
||
|
|
"drift/rejected_abs": 0.1704963594675064,
|
||
|
|
"epoch": 0.5446076675026872,
|
||
|
|
"grad_norm": 316.0,
|
||
|
|
"learning_rate": 2.184502577160426e-07,
|
||
|
|
"logits/chosen": -2.07682466506958,
|
||
|
|
"logits/rejected": -2.1273488998413086,
|
||
|
|
"logps/chosen": -0.286944180727005,
|
||
|
|
"logps/rejected": -0.2869289517402649,
|
||
|
|
"loss": 23.644794464111328,
|
||
|
|
"loss/core": 23.60843276977539,
|
||
|
|
"loss/preference_sft": 0.286944180727005,
|
||
|
|
"loss/reference_anchor": 0.4561174809932709,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.3612422943115234,
|
||
|
|
"rewards/margins": 1.6572866439819336,
|
||
|
|
"rewards/rejected": 1.703955888748169,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18439261615276337,
|
||
|
|
"drift/rejected_abs": 0.0657983049750328,
|
||
|
|
"epoch": 0.5493849277439389,
|
||
|
|
"grad_norm": 43.0,
|
||
|
|
"learning_rate": 2.1304764721759732e-07,
|
||
|
|
"logits/chosen": -2.1571121215820312,
|
||
|
|
"logits/rejected": -2.188605546951294,
|
||
|
|
"logps/chosen": -0.3006740212440491,
|
||
|
|
"logps/rejected": -0.32234296202659607,
|
||
|
|
"loss": 24.048736572265625,
|
||
|
|
"loss/core": 24.03409194946289,
|
||
|
|
"loss/preference_sft": 0.3006740212440491,
|
||
|
|
"loss/reference_anchor": 0.1651538759469986,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.8426148891448975,
|
||
|
|
"rewards/margins": 1.273414134979248,
|
||
|
|
"rewards/rejected": 0.569200873374939,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14222335815429688,
|
||
|
|
"drift/rejected_abs": 0.09565664827823639,
|
||
|
|
"epoch": 0.5541621879851905,
|
||
|
|
"grad_norm": 30.25,
|
||
|
|
"learning_rate": 2.0766262403774385e-07,
|
||
|
|
"logits/chosen": -2.4316046237945557,
|
||
|
|
"logits/rejected": -2.4871370792388916,
|
||
|
|
"logps/chosen": -0.26630157232284546,
|
||
|
|
"logps/rejected": -0.2813344895839691,
|
||
|
|
"loss": 24.60654067993164,
|
||
|
|
"loss/core": 24.598276138305664,
|
||
|
|
"loss/preference_sft": 0.26630157232284546,
|
||
|
|
"loss/reference_anchor": 0.08357779681682587,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.4222335815429688,
|
||
|
|
"rewards/margins": 0.46712565422058105,
|
||
|
|
"rewards/rejected": 0.9551078677177429,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22317564487457275,
|
||
|
|
"drift/rejected_abs": 0.13778264820575714,
|
||
|
|
"epoch": 0.5589394482264421,
|
||
|
|
"grad_norm": 68.0,
|
||
|
|
"learning_rate": 2.0229775115586378e-07,
|
||
|
|
"logits/chosen": -2.1374573707580566,
|
||
|
|
"logits/rejected": -2.1010804176330566,
|
||
|
|
"logps/chosen": -0.26939648389816284,
|
||
|
|
"logps/rejected": -0.2822672426700592,
|
||
|
|
"loss": 24.302589416503906,
|
||
|
|
"loss/core": 24.24777603149414,
|
||
|
|
"loss/preference_sft": 0.26939648389816284,
|
||
|
|
"loss/reference_anchor": 0.7039062976837158,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.230914354324341,
|
||
|
|
"rewards/margins": 0.8574030995368958,
|
||
|
|
"rewards/rejected": 1.3735109567642212,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3644371032714844,
|
||
|
|
"drift/rejected_abs": 0.104056216776371,
|
||
|
|
"epoch": 0.5637167084676937,
|
||
|
|
"grad_norm": 51.0,
|
||
|
|
"learning_rate": 1.9695558196088844e-07,
|
||
|
|
"logits/chosen": -2.0276551246643066,
|
||
|
|
"logits/rejected": -2.1235756874084473,
|
||
|
|
"logps/chosen": -0.2852238714694977,
|
||
|
|
"logps/rejected": -0.2887842059135437,
|
||
|
|
"loss": 23.151445388793945,
|
||
|
|
"loss/core": 23.115150451660156,
|
||
|
|
"loss/preference_sft": 0.2852238714694977,
|
||
|
|
"loss/reference_anchor": 0.45538654923439026,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.6441140174865723,
|
||
|
|
"rewards/margins": 2.6299378871917725,
|
||
|
|
"rewards/rejected": 1.0141762495040894,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1715865433216095,
|
||
|
|
"drift/rejected_abs": 0.09020796418190002,
|
||
|
|
"epoch": 0.5684939687089454,
|
||
|
|
"grad_norm": 69.5,
|
||
|
|
"learning_rate": 1.9163865903602372e-07,
|
||
|
|
"logits/chosen": -2.0477294921875,
|
||
|
|
"logits/rejected": -2.0416600704193115,
|
||
|
|
"logps/chosen": -0.2748182415962219,
|
||
|
|
"logps/rejected": -0.27176275849342346,
|
||
|
|
"loss": 24.21047592163086,
|
||
|
|
"loss/core": 24.204145431518555,
|
||
|
|
"loss/preference_sft": 0.2748182415962219,
|
||
|
|
"loss/reference_anchor": 0.05691518262028694,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7147114276885986,
|
||
|
|
"rewards/margins": 0.8242376446723938,
|
||
|
|
"rewards/rejected": 0.8904738426208496,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18755662441253662,
|
||
|
|
"drift/rejected_abs": 0.10950811952352524,
|
||
|
|
"epoch": 0.5732712289501971,
|
||
|
|
"grad_norm": 103.0,
|
||
|
|
"learning_rate": 1.8634951294861806e-07,
|
||
|
|
"logits/chosen": -2.2788078784942627,
|
||
|
|
"logits/rejected": -2.2681713104248047,
|
||
|
|
"logps/chosen": -0.266190767288208,
|
||
|
|
"logps/rejected": -0.2825302183628082,
|
||
|
|
"loss": 24.342437744140625,
|
||
|
|
"loss/core": 24.329063415527344,
|
||
|
|
"loss/preference_sft": 0.266190767288208,
|
||
|
|
"loss/reference_anchor": 0.15168902277946472,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.8744417428970337,
|
||
|
|
"rewards/margins": 0.7936787009239197,
|
||
|
|
"rewards/rejected": 1.0807631015777588,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.42088302969932556,
|
||
|
|
"drift/rejected_abs": 0.1206652969121933,
|
||
|
|
"epoch": 0.5780484891914487,
|
||
|
|
"grad_norm": 122.5,
|
||
|
|
"learning_rate": 1.810906610457502e-07,
|
||
|
|
"logits/chosen": -2.1177077293395996,
|
||
|
|
"logits/rejected": -2.1916587352752686,
|
||
|
|
"logps/chosen": -0.2778300940990448,
|
||
|
|
"logps/rejected": -0.275773286819458,
|
||
|
|
"loss": 23.372730255126953,
|
||
|
|
"loss/core": 23.302967071533203,
|
||
|
|
"loss/preference_sft": 0.2778300940990448,
|
||
|
|
"loss/reference_anchor": 0.9023693203926086,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 4.206851482391357,
|
||
|
|
"rewards/margins": 3.001718759536743,
|
||
|
|
"rewards/rejected": 1.2051331996917725,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4304037094116211,
|
||
|
|
"drift/rejected_abs": 0.19502094388008118,
|
||
|
|
"epoch": 0.5828257494327004,
|
||
|
|
"grad_norm": 145.0,
|
||
|
|
"learning_rate": 1.7586460625610726e-07,
|
||
|
|
"logits/chosen": -2.118242025375366,
|
||
|
|
"logits/rejected": -2.1343255043029785,
|
||
|
|
"logps/chosen": -0.27321526408195496,
|
||
|
|
"logps/rejected": -0.27437523007392883,
|
||
|
|
"loss": 23.397014617919922,
|
||
|
|
"loss/core": 23.323720932006836,
|
||
|
|
"loss/preference_sft": 0.27321526408195496,
|
||
|
|
"loss/reference_anchor": 0.9498865008354187,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 4.302402973175049,
|
||
|
|
"rewards/margins": 2.3571293354034424,
|
||
|
|
"rewards/rejected": 1.945273995399475,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27419549226760864,
|
||
|
|
"drift/rejected_abs": 0.1494186818599701,
|
||
|
|
"epoch": 0.587603009673952,
|
||
|
|
"grad_norm": 69.0,
|
||
|
|
"learning_rate": 1.70673835898727e-07,
|
||
|
|
"logits/chosen": -1.967576026916504,
|
||
|
|
"logits/rejected": -2.034571409225464,
|
||
|
|
"logps/chosen": -0.3140396773815155,
|
||
|
|
"logps/rejected": -0.3035440444946289,
|
||
|
|
"loss": 23.98012351989746,
|
||
|
|
"loss/core": 23.95610237121582,
|
||
|
|
"loss/preference_sft": 0.3140396773815155,
|
||
|
|
"loss/reference_anchor": 0.28889569640159607,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.739633321762085,
|
||
|
|
"rewards/margins": 1.2492014169692993,
|
||
|
|
"rewards/rejected": 1.4904316663742065,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16910451650619507,
|
||
|
|
"drift/rejected_abs": 0.14680986106395721,
|
||
|
|
"epoch": 0.5923802699152036,
|
||
|
|
"grad_norm": 61.25,
|
||
|
|
"learning_rate": 1.6552082049916824e-07,
|
||
|
|
"logits/chosen": -2.143625259399414,
|
||
|
|
"logits/rejected": -2.1699137687683105,
|
||
|
|
"logps/chosen": -0.286045640707016,
|
||
|
|
"logps/rejected": -0.2911421060562134,
|
||
|
|
"loss": 25.047000885009766,
|
||
|
|
"loss/core": 25.030277252197266,
|
||
|
|
"loss/preference_sft": 0.286045640707016,
|
||
|
|
"loss/reference_anchor": 0.19438524544239044,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.690314531326294,
|
||
|
|
"rewards/margins": 0.22506046295166016,
|
||
|
|
"rewards/rejected": 1.4652539491653442,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.388736367225647,
|
||
|
|
"drift/rejected_abs": 0.1826535165309906,
|
||
|
|
"epoch": 0.5971575301564552,
|
||
|
|
"grad_norm": 95.5,
|
||
|
|
"learning_rate": 1.6040801261367493e-07,
|
||
|
|
"logits/chosen": -1.9324958324432373,
|
||
|
|
"logits/rejected": -1.961570143699646,
|
||
|
|
"logps/chosen": -0.304928719997406,
|
||
|
|
"logps/rejected": -0.33531856536865234,
|
||
|
|
"loss": 24.033096313476562,
|
||
|
|
"loss/core": 23.950403213500977,
|
||
|
|
"loss/preference_sft": 0.304928719997406,
|
||
|
|
"loss/reference_anchor": 1.0720922946929932,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.885380268096924,
|
||
|
|
"rewards/margins": 2.3656325340270996,
|
||
|
|
"rewards/rejected": 1.5197473764419556,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29494503140449524,
|
||
|
|
"drift/rejected_abs": 0.12427574396133423,
|
||
|
|
"epoch": 0.6019347903977069,
|
||
|
|
"grad_norm": 57.0,
|
||
|
|
"learning_rate": 1.5533784566189175e-07,
|
||
|
|
"logits/chosen": -1.9347823858261108,
|
||
|
|
"logits/rejected": -2.0498507022857666,
|
||
|
|
"logps/chosen": -0.27183371782302856,
|
||
|
|
"logps/rejected": -0.2836715281009674,
|
||
|
|
"loss": 23.67662239074707,
|
||
|
|
"loss/core": 23.64113998413086,
|
||
|
|
"loss/preference_sft": 0.27183371782302856,
|
||
|
|
"loss/reference_anchor": 0.4459313452243805,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.949350595474243,
|
||
|
|
"rewards/margins": 1.7082860469818115,
|
||
|
|
"rewards/rejected": 1.2410645484924316,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23291683197021484,
|
||
|
|
"drift/rejected_abs": 0.10206238925457001,
|
||
|
|
"epoch": 0.6067120506389586,
|
||
|
|
"grad_norm": 222.0,
|
||
|
|
"learning_rate": 1.5031273276868845e-07,
|
||
|
|
"logits/chosen": -1.984095573425293,
|
||
|
|
"logits/rejected": -2.0696752071380615,
|
||
|
|
"logps/chosen": -0.3027849495410919,
|
||
|
|
"logps/rejected": -0.2994707524776459,
|
||
|
|
"loss": 23.96482276916504,
|
||
|
|
"loss/core": 23.94552230834961,
|
||
|
|
"loss/preference_sft": 0.3027849495410919,
|
||
|
|
"loss/reference_anchor": 0.2270624190568924,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.327078342437744,
|
||
|
|
"rewards/margins": 1.3064545392990112,
|
||
|
|
"rewards/rejected": 1.0206239223480225,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1955791413784027,
|
||
|
|
"drift/rejected_abs": 0.08006999641656876,
|
||
|
|
"epoch": 0.6114893108802102,
|
||
|
|
"grad_norm": 49.5,
|
||
|
|
"learning_rate": 1.4533506561564305e-07,
|
||
|
|
"logits/chosen": -2.2103934288024902,
|
||
|
|
"logits/rejected": -2.2860031127929688,
|
||
|
|
"logps/chosen": -0.3017186224460602,
|
||
|
|
"logps/rejected": -0.2985337972640991,
|
||
|
|
"loss": 24.067489624023438,
|
||
|
|
"loss/core": 24.050811767578125,
|
||
|
|
"loss/preference_sft": 0.3017186224460602,
|
||
|
|
"loss/reference_anchor": 0.19217993319034576,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.9552767276763916,
|
||
|
|
"rewards/margins": 1.1555898189544678,
|
||
|
|
"rewards/rejected": 0.7996870279312134,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16859070956707,
|
||
|
|
"drift/rejected_abs": 0.12346317619085312,
|
||
|
|
"epoch": 0.6162665711214619,
|
||
|
|
"grad_norm": 117.5,
|
||
|
|
"learning_rate": 1.404072133027306e-07,
|
||
|
|
"logits/chosen": -2.1554760932922363,
|
||
|
|
"logits/rejected": -2.13128662109375,
|
||
|
|
"logps/chosen": -0.2895345687866211,
|
||
|
|
"logps/rejected": -0.2951858341693878,
|
||
|
|
"loss": 24.680572509765625,
|
||
|
|
"loss/core": 24.669631958007812,
|
||
|
|
"loss/preference_sft": 0.2895345687866211,
|
||
|
|
"loss/reference_anchor": 0.11693717539310455,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.6848987340927124,
|
||
|
|
"rewards/margins": 0.4532623291015625,
|
||
|
|
"rewards/rejected": 1.2316365242004395,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13376560807228088,
|
||
|
|
"drift/rejected_abs": 0.08641010522842407,
|
||
|
|
"epoch": 0.6210438313627135,
|
||
|
|
"grad_norm": 66.5,
|
||
|
|
"learning_rate": 1.3553152122076078e-07,
|
||
|
|
"logits/chosen": -2.2198781967163086,
|
||
|
|
"logits/rejected": -2.170907974243164,
|
||
|
|
"logps/chosen": -0.28950920701026917,
|
||
|
|
"logps/rejected": -0.2986505925655365,
|
||
|
|
"loss": 24.61799430847168,
|
||
|
|
"loss/core": 24.610118865966797,
|
||
|
|
"loss/preference_sft": 0.28950920701026917,
|
||
|
|
"loss/reference_anchor": 0.07604795694351196,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.3332895040512085,
|
||
|
|
"rewards/margins": 0.4888167381286621,
|
||
|
|
"rewards/rejected": 0.8444727063179016,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2010502815246582,
|
||
|
|
"drift/rejected_abs": 0.0992017537355423,
|
||
|
|
"epoch": 0.6258210916039652,
|
||
|
|
"grad_norm": 304.0,
|
||
|
|
"learning_rate": 1.3071030993509787e-07,
|
||
|
|
"logits/chosen": -2.059145450592041,
|
||
|
|
"logits/rejected": -2.1219570636749268,
|
||
|
|
"logps/chosen": -0.2817350924015045,
|
||
|
|
"logps/rejected": -0.28943654894828796,
|
||
|
|
"loss": 24.06000328063965,
|
||
|
|
"loss/core": 24.04839515686035,
|
||
|
|
"loss/preference_sft": 0.2817350924015045,
|
||
|
|
"loss/reference_anchor": 0.12662085890769958,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.009782314300537,
|
||
|
|
"rewards/margins": 1.0185507535934448,
|
||
|
|
"rewards/rejected": 0.9912316203117371,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2789165675640106,
|
||
|
|
"drift/rejected_abs": 0.1331566572189331,
|
||
|
|
"epoch": 0.6305983518452167,
|
||
|
|
"grad_norm": 45.75,
|
||
|
|
"learning_rate": 1.2594587408119804e-07,
|
||
|
|
"logits/chosen": -2.0298891067504883,
|
||
|
|
"logits/rejected": -2.0245847702026367,
|
||
|
|
"logps/chosen": -0.29279690980911255,
|
||
|
|
"logps/rejected": -0.30883678793907166,
|
||
|
|
"loss": 24.24443244934082,
|
||
|
|
"loss/core": 24.194019317626953,
|
||
|
|
"loss/preference_sft": 0.29279690980911255,
|
||
|
|
"loss/reference_anchor": 0.6429013609886169,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.7881667613983154,
|
||
|
|
"rewards/margins": 1.5086616277694702,
|
||
|
|
"rewards/rejected": 1.2795050144195557,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23610401153564453,
|
||
|
|
"drift/rejected_abs": 0.1503429114818573,
|
||
|
|
"epoch": 0.6353756120864684,
|
||
|
|
"grad_norm": 123.5,
|
||
|
|
"learning_rate": 1.2124048127248644e-07,
|
||
|
|
"logits/chosen": -2.188072681427002,
|
||
|
|
"logits/rejected": -2.164827823638916,
|
||
|
|
"logps/chosen": -0.28306499123573303,
|
||
|
|
"logps/rejected": -0.27927109599113464,
|
||
|
|
"loss": 24.441707611083984,
|
||
|
|
"loss/core": 24.4201717376709,
|
||
|
|
"loss/preference_sft": 0.28306499123573303,
|
||
|
|
"loss/reference_anchor": 0.25882014632225037,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.3602824211120605,
|
||
|
|
"rewards/margins": 0.8587505221366882,
|
||
|
|
"rewards/rejected": 1.5015321969985962,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3666432499885559,
|
||
|
|
"drift/rejected_abs": 0.1319333016872406,
|
||
|
|
"epoch": 0.64015287232772,
|
||
|
|
"grad_norm": 44.0,
|
||
|
|
"learning_rate": 1.1659637102109712e-07,
|
||
|
|
"logits/chosen": -1.9824602603912354,
|
||
|
|
"logits/rejected": -2.0293171405792236,
|
||
|
|
"logps/chosen": -0.2721920907497406,
|
||
|
|
"logps/rejected": -0.31627121567726135,
|
||
|
|
"loss": 23.281818389892578,
|
||
|
|
"loss/core": 23.24245834350586,
|
||
|
|
"loss/preference_sft": 0.2721920907497406,
|
||
|
|
"loss/reference_anchor": 0.49754709005355835,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.666236162185669,
|
||
|
|
"rewards/margins": 2.3700363636016846,
|
||
|
|
"rewards/rejected": 1.2962002754211426,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.162507563829422,
|
||
|
|
"drift/rejected_abs": 0.09013229608535767,
|
||
|
|
"epoch": 0.6449301325689717,
|
||
|
|
"grad_norm": 93.0,
|
||
|
|
"learning_rate": 1.1201575367198546e-07,
|
||
|
|
"logits/chosen": -2.038818359375,
|
||
|
|
"logits/rejected": -2.0799050331115723,
|
||
|
|
"logps/chosen": -0.28820866346359253,
|
||
|
|
"logps/rejected": -0.28691327571868896,
|
||
|
|
"loss": 24.332555770874023,
|
||
|
|
"loss/core": 24.325624465942383,
|
||
|
|
"loss/preference_sft": 0.28820866346359253,
|
||
|
|
"loss/reference_anchor": 0.06361403316259384,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6243938207626343,
|
||
|
|
"rewards/margins": 0.730514645576477,
|
||
|
|
"rewards/rejected": 0.893879234790802,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2139333039522171,
|
||
|
|
"drift/rejected_abs": 0.13963082432746887,
|
||
|
|
"epoch": 0.6497073928102234,
|
||
|
|
"grad_norm": 43.75,
|
||
|
|
"learning_rate": 1.0750080935092423e-07,
|
||
|
|
"logits/chosen": -2.1613402366638184,
|
||
|
|
"logits/rejected": -2.079357385635376,
|
||
|
|
"logps/chosen": -0.28284314274787903,
|
||
|
|
"logps/rejected": -0.2906249463558197,
|
||
|
|
"loss": 24.423181533813477,
|
||
|
|
"loss/core": 24.4051456451416,
|
||
|
|
"loss/preference_sft": 0.28284314274787903,
|
||
|
|
"loss/reference_anchor": 0.2122153490781784,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1380627155303955,
|
||
|
|
"rewards/margins": 0.7430015802383423,
|
||
|
|
"rewards/rejected": 1.3950612545013428,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28872284293174744,
|
||
|
|
"drift/rejected_abs": 0.15741479396820068,
|
||
|
|
"epoch": 0.654484653051475,
|
||
|
|
"grad_norm": 57.25,
|
||
|
|
"learning_rate": 1.0305368692688174e-07,
|
||
|
|
"logits/chosen": -2.088265895843506,
|
||
|
|
"logits/rejected": -2.072359561920166,
|
||
|
|
"logps/chosen": -0.253498375415802,
|
||
|
|
"logps/rejected": -0.2732449471950531,
|
||
|
|
"loss": 24.38766098022461,
|
||
|
|
"loss/core": 24.349506378173828,
|
||
|
|
"loss/preference_sft": 0.253498375415802,
|
||
|
|
"loss/reference_anchor": 0.4833889901638031,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.8872287273406982,
|
||
|
|
"rewards/margins": 1.3145825862884521,
|
||
|
|
"rewards/rejected": 1.5726460218429565,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17745468020439148,
|
||
|
|
"drift/rejected_abs": 0.1100490465760231,
|
||
|
|
"epoch": 0.6592619132927267,
|
||
|
|
"grad_norm": 51.5,
|
||
|
|
"learning_rate": 9.867650298927643e-08,
|
||
|
|
"logits/chosen": -1.8866245746612549,
|
||
|
|
"logits/rejected": -1.9085317850112915,
|
||
|
|
"logps/chosen": -0.307719886302948,
|
||
|
|
"logps/rejected": -0.32763782143592834,
|
||
|
|
"loss": 24.435400009155273,
|
||
|
|
"loss/core": 24.425445556640625,
|
||
|
|
"loss/preference_sft": 0.307719886302948,
|
||
|
|
"loss/reference_anchor": 0.10196495056152344,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7728378772735596,
|
||
|
|
"rewards/margins": 0.6770357489585876,
|
||
|
|
"rewards/rejected": 1.0958020687103271,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1973785161972046,
|
||
|
|
"drift/rejected_abs": 0.06951657682657242,
|
||
|
|
"epoch": 0.6640391735339782,
|
||
|
|
"grad_norm": 65.5,
|
||
|
|
"learning_rate": 9.437134084059515e-08,
|
||
|
|
"logits/chosen": -2.272217273712158,
|
||
|
|
"logits/rejected": -2.280867576599121,
|
||
|
|
"logps/chosen": -0.3060583770275116,
|
||
|
|
"logps/rejected": -0.29238781332969666,
|
||
|
|
"loss": 24.02126121520996,
|
||
|
|
"loss/core": 24.004791259765625,
|
||
|
|
"loss/preference_sft": 0.3060583770275116,
|
||
|
|
"loss/reference_anchor": 0.18900753557682037,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.972781777381897,
|
||
|
|
"rewards/margins": 1.285192847251892,
|
||
|
|
"rewards/rejected": 0.6875890493392944,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22317072749137878,
|
||
|
|
"drift/rejected_abs": 0.16237227618694305,
|
||
|
|
"epoch": 0.6688164337752299,
|
||
|
|
"grad_norm": 1552.0,
|
||
|
|
"learning_rate": 9.014024950485383e-08,
|
||
|
|
"logits/chosen": -2.1077773571014404,
|
||
|
|
"logits/rejected": -2.214353561401367,
|
||
|
|
"logps/chosen": -0.29673534631729126,
|
||
|
|
"logps/rejected": -0.3178013861179352,
|
||
|
|
"loss": 24.56673240661621,
|
||
|
|
"loss/core": 24.545368194580078,
|
||
|
|
"loss/preference_sft": 0.29673534631729126,
|
||
|
|
"loss/reference_anchor": 0.25515273213386536,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.231229305267334,
|
||
|
|
"rewards/margins": 0.6277844309806824,
|
||
|
|
"rewards/rejected": 1.603445053100586,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.341328501701355,
|
||
|
|
"drift/rejected_abs": 0.15423105657100677,
|
||
|
|
"epoch": 0.6735936940164815,
|
||
|
|
"grad_norm": 132.0,
|
||
|
|
"learning_rate": 8.598524275237321e-08,
|
||
|
|
"logits/chosen": -2.1965928077697754,
|
||
|
|
"logits/rejected": -2.154327630996704,
|
||
|
|
"logps/chosen": -0.27952122688293457,
|
||
|
|
"logps/rejected": -0.2664346396923065,
|
||
|
|
"loss": 23.59442138671875,
|
||
|
|
"loss/core": 23.561824798583984,
|
||
|
|
"loss/preference_sft": 0.27952122688293457,
|
||
|
|
"loss/reference_anchor": 0.40662726759910583,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.412951946258545,
|
||
|
|
"rewards/margins": 1.871299386024475,
|
||
|
|
"rewards/rejected": 1.5416524410247803,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3609875440597534,
|
||
|
|
"drift/rejected_abs": 0.19022467732429504,
|
||
|
|
"epoch": 0.6783709542577332,
|
||
|
|
"grad_norm": 576.0,
|
||
|
|
"learning_rate": 8.190829814133293e-08,
|
||
|
|
"logits/chosen": -1.8862717151641846,
|
||
|
|
"logits/rejected": -1.87451171875,
|
||
|
|
"logps/chosen": -0.2760660648345947,
|
||
|
|
"logps/rejected": -0.2974444329738617,
|
||
|
|
"loss": 23.68022346496582,
|
||
|
|
"loss/core": 23.627628326416016,
|
||
|
|
"loss/preference_sft": 0.2760660648345947,
|
||
|
|
"loss/reference_anchor": 0.6736692786216736,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.608205795288086,
|
||
|
|
"rewards/margins": 1.707396149635315,
|
||
|
|
"rewards/rejected": 1.90080988407135,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18668507039546967,
|
||
|
|
"drift/rejected_abs": 0.07842984795570374,
|
||
|
|
"epoch": 0.6831482144989849,
|
||
|
|
"grad_norm": 57.25,
|
||
|
|
"learning_rate": 7.791135607656146e-08,
|
||
|
|
"logits/chosen": -2.2538669109344482,
|
||
|
|
"logits/rejected": -2.2352232933044434,
|
||
|
|
"logps/chosen": -0.28401559591293335,
|
||
|
|
"logps/rejected": -0.3294680714607239,
|
||
|
|
"loss": 24.112934112548828,
|
||
|
|
"loss/core": 24.091781616210938,
|
||
|
|
"loss/preference_sft": 0.28401559591293335,
|
||
|
|
"loss/reference_anchor": 0.25365859270095825,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8665748834609985,
|
||
|
|
"rewards/margins": 1.0951749086380005,
|
||
|
|
"rewards/rejected": 0.771399974822998,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21051010489463806,
|
||
|
|
"drift/rejected_abs": 0.1500566005706787,
|
||
|
|
"epoch": 0.6879254747402365,
|
||
|
|
"grad_norm": 136.0,
|
||
|
|
"learning_rate": 7.399631888600796e-08,
|
||
|
|
"logits/chosen": -1.9951555728912354,
|
||
|
|
"logits/rejected": -2.00339674949646,
|
||
|
|
"logps/chosen": -0.2962464392185211,
|
||
|
|
"logps/rejected": -0.30031222105026245,
|
||
|
|
"loss": 24.666812896728516,
|
||
|
|
"loss/core": 24.645822525024414,
|
||
|
|
"loss/preference_sft": 0.2962464392185211,
|
||
|
|
"loss/reference_anchor": 0.25020235776901245,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.1030123233795166,
|
||
|
|
"rewards/margins": 0.6037241816520691,
|
||
|
|
"rewards/rejected": 1.4992883205413818,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15870442986488342,
|
||
|
|
"drift/rejected_abs": 0.07348223030567169,
|
||
|
|
"epoch": 0.6927027349814882,
|
||
|
|
"grad_norm": 66.5,
|
||
|
|
"learning_rate": 7.016504991533726e-08,
|
||
|
|
"logits/chosen": -2.1241562366485596,
|
||
|
|
"logits/rejected": -2.2400271892547607,
|
||
|
|
"logps/chosen": -0.28562217950820923,
|
||
|
|
"logps/rejected": -0.2771227955818176,
|
||
|
|
"loss": 24.232282638549805,
|
||
|
|
"loss/core": 24.223468780517578,
|
||
|
|
"loss/preference_sft": 0.28562217950820923,
|
||
|
|
"loss/reference_anchor": 0.08894944936037064,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.5855097770690918,
|
||
|
|
"rewards/margins": 0.8525517582893372,
|
||
|
|
"rewards/rejected": 0.7329580783843994,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.102408766746521,
|
||
|
|
"drift/rejected_abs": 0.08807560056447983,
|
||
|
|
"epoch": 0.6974799952227397,
|
||
|
|
"grad_norm": 956.0,
|
||
|
|
"learning_rate": 6.641937264107867e-08,
|
||
|
|
"logits/chosen": -2.2225677967071533,
|
||
|
|
"logits/rejected": -2.271383285522461,
|
||
|
|
"logps/chosen": -0.28616029024124146,
|
||
|
|
"logps/rejected": -0.293459951877594,
|
||
|
|
"loss": 24.99797821044922,
|
||
|
|
"loss/core": 24.98850440979004,
|
||
|
|
"loss/preference_sft": 0.28616029024124146,
|
||
|
|
"loss/reference_anchor": 0.09770476073026657,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.022210717201233,
|
||
|
|
"rewards/margins": 0.14174190163612366,
|
||
|
|
"rewards/rejected": 0.8804686665534973,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1421697437763214,
|
||
|
|
"drift/rejected_abs": 0.06707651913166046,
|
||
|
|
"epoch": 0.7022572554639914,
|
||
|
|
"grad_norm": 119.0,
|
||
|
|
"learning_rate": 6.276106980274944e-08,
|
||
|
|
"logits/chosen": -2.0496909618377686,
|
||
|
|
"logits/rejected": -2.097294330596924,
|
||
|
|
"logps/chosen": -0.31965214014053345,
|
||
|
|
"logps/rejected": -0.31716209650039673,
|
||
|
|
"loss": 24.297515869140625,
|
||
|
|
"loss/core": 24.289684295654297,
|
||
|
|
"loss/preference_sft": 0.31965214014053345,
|
||
|
|
"loss/reference_anchor": 0.07245230674743652,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.4212677478790283,
|
||
|
|
"rewards/margins": 0.7517526149749756,
|
||
|
|
"rewards/rejected": 0.6695151329040527,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2564278841018677,
|
||
|
|
"drift/rejected_abs": 0.18303130567073822,
|
||
|
|
"epoch": 0.707034515705243,
|
||
|
|
"grad_norm": 71.0,
|
||
|
|
"learning_rate": 5.919188255436777e-08,
|
||
|
|
"logits/chosen": -2.1012163162231445,
|
||
|
|
"logits/rejected": -2.1193010807037354,
|
||
|
|
"logps/chosen": -0.3130371868610382,
|
||
|
|
"logps/rejected": -0.3160560727119446,
|
||
|
|
"loss": 25.59921646118164,
|
||
|
|
"loss/core": 25.540393829345703,
|
||
|
|
"loss/preference_sft": 0.3130371868610382,
|
||
|
|
"loss/reference_anchor": 0.7530360221862793,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.564279079437256,
|
||
|
|
"rewards/margins": 0.7340623736381531,
|
||
|
|
"rewards/rejected": 1.830216646194458,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3141566514968872,
|
||
|
|
"drift/rejected_abs": 0.15219292044639587,
|
||
|
|
"epoch": 0.7118117759464947,
|
||
|
|
"grad_norm": 126.0,
|
||
|
|
"learning_rate": 5.571350963575727e-08,
|
||
|
|
"logits/chosen": -2.0531277656555176,
|
||
|
|
"logits/rejected": -2.09647798538208,
|
||
|
|
"logps/chosen": -0.2884739339351654,
|
||
|
|
"logps/rejected": -0.2817172408103943,
|
||
|
|
"loss": 23.735965728759766,
|
||
|
|
"loss/core": 23.71042823791504,
|
||
|
|
"loss/preference_sft": 0.2884739339351654,
|
||
|
|
"loss/reference_anchor": 0.3116436302661896,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.141024351119995,
|
||
|
|
"rewards/margins": 1.6229345798492432,
|
||
|
|
"rewards/rejected": 1.5180898904800415,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12072908878326416,
|
||
|
|
"drift/rejected_abs": 0.12682561576366425,
|
||
|
|
"epoch": 0.7165890361877463,
|
||
|
|
"grad_norm": 340.0,
|
||
|
|
"learning_rate": 5.232760656403923e-08,
|
||
|
|
"logits/chosen": -2.1889564990997314,
|
||
|
|
"logits/rejected": -2.1800601482391357,
|
||
|
|
"logps/chosen": -0.3116965889930725,
|
||
|
|
"logps/rejected": -0.3028327226638794,
|
||
|
|
"loss": 25.507654190063477,
|
||
|
|
"loss/core": 25.483829498291016,
|
||
|
|
"loss/preference_sft": 0.3116965889930725,
|
||
|
|
"loss/reference_anchor": 0.28652527928352356,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.2060551643371582,
|
||
|
|
"rewards/margins": -0.05827074497938156,
|
||
|
|
"rewards/rejected": 1.2643258571624756,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24006719887256622,
|
||
|
|
"drift/rejected_abs": 0.17291317880153656,
|
||
|
|
"epoch": 0.721366296428998,
|
||
|
|
"grad_norm": 133.0,
|
||
|
|
"learning_rate": 4.9035784845695675e-08,
|
||
|
|
"logits/chosen": -1.9279276132583618,
|
||
|
|
"logits/rejected": -1.8381156921386719,
|
||
|
|
"logps/chosen": -0.3074774742126465,
|
||
|
|
"logps/rejected": -0.29301613569259644,
|
||
|
|
"loss": 24.52367401123047,
|
||
|
|
"loss/core": 24.506526947021484,
|
||
|
|
"loss/preference_sft": 0.3074774742126465,
|
||
|
|
"loss/reference_anchor": 0.19787776470184326,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.3972103595733643,
|
||
|
|
"rewards/margins": 0.6746050715446472,
|
||
|
|
"rewards/rejected": 1.7226054668426514,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19889570772647858,
|
||
|
|
"drift/rejected_abs": 0.08718482404947281,
|
||
|
|
"epoch": 0.7261435566702497,
|
||
|
|
"grad_norm": 64.0,
|
||
|
|
"learning_rate": 4.583961120958027e-08,
|
||
|
|
"logits/chosen": -2.186131715774536,
|
||
|
|
"logits/rejected": -2.203399419784546,
|
||
|
|
"logps/chosen": -0.30299732089042664,
|
||
|
|
"logps/rejected": -0.29353067278862,
|
||
|
|
"loss": 24.01272964477539,
|
||
|
|
"loss/core": 24.000045776367188,
|
||
|
|
"loss/preference_sft": 0.30299732089042664,
|
||
|
|
"loss/reference_anchor": 0.13879242539405823,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9878475666046143,
|
||
|
|
"rewards/margins": 1.1234432458877563,
|
||
|
|
"rewards/rejected": 0.8644043803215027,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1797488033771515,
|
||
|
|
"drift/rejected_abs": 0.1066058874130249,
|
||
|
|
"epoch": 0.7309208169115012,
|
||
|
|
"grad_norm": 106.5,
|
||
|
|
"learning_rate": 4.2740606861239594e-08,
|
||
|
|
"logits/chosen": -2.08768892288208,
|
||
|
|
"logits/rejected": -2.158001184463501,
|
||
|
|
"logps/chosen": -0.275774210691452,
|
||
|
|
"logps/rejected": -0.27186232805252075,
|
||
|
|
"loss": 24.34197425842285,
|
||
|
|
"loss/core": 24.330942153930664,
|
||
|
|
"loss/preference_sft": 0.275774210691452,
|
||
|
|
"loss/reference_anchor": 0.11952831596136093,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7971556186676025,
|
||
|
|
"rewards/margins": 0.7334049344062805,
|
||
|
|
"rewards/rejected": 1.0637508630752563,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3747478127479553,
|
||
|
|
"drift/rejected_abs": 0.2356497049331665,
|
||
|
|
"epoch": 0.7356980771527529,
|
||
|
|
"grad_norm": 222.0,
|
||
|
|
"learning_rate": 3.974024675890189e-08,
|
||
|
|
"logits/chosen": -1.9823691844940186,
|
||
|
|
"logits/rejected": -1.9497228860855103,
|
||
|
|
"logps/chosen": -0.3086339831352234,
|
||
|
|
"logps/rejected": -0.28454282879829407,
|
||
|
|
"loss": 24.525297164916992,
|
||
|
|
"loss/core": 24.43834114074707,
|
||
|
|
"loss/preference_sft": 0.3086339831352234,
|
||
|
|
"loss/reference_anchor": 1.1285443305969238,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.744788408279419,
|
||
|
|
"rewards/margins": 1.3908659219741821,
|
||
|
|
"rewards/rejected": 2.3539223670959473,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15283748507499695,
|
||
|
|
"drift/rejected_abs": 0.09471901506185532,
|
||
|
|
"epoch": 0.7404753373940045,
|
||
|
|
"grad_norm": 32.75,
|
||
|
|
"learning_rate": 3.683995891147695e-08,
|
||
|
|
"logits/chosen": -2.0964419841766357,
|
||
|
|
"logits/rejected": -2.1264514923095703,
|
||
|
|
"logps/chosen": -0.30610623955726624,
|
||
|
|
"logps/rejected": -0.29977482557296753,
|
||
|
|
"loss": 24.4835147857666,
|
||
|
|
"loss/core": 24.475711822509766,
|
||
|
|
"loss/preference_sft": 0.30610623955726624,
|
||
|
|
"loss/reference_anchor": 0.0734710842370987,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.5278440713882446,
|
||
|
|
"rewards/margins": 0.5806539058685303,
|
||
|
|
"rewards/rejected": 0.9471901059150696,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1896996945142746,
|
||
|
|
"drift/rejected_abs": 0.13039933145046234,
|
||
|
|
"epoch": 0.7452525976352562,
|
||
|
|
"grad_norm": 102.5,
|
||
|
|
"learning_rate": 3.404112369890108e-08,
|
||
|
|
"logits/chosen": -2.1795246601104736,
|
||
|
|
"logits/rejected": -2.1940202713012695,
|
||
|
|
"logps/chosen": -0.3016170859336853,
|
||
|
|
"logps/rejected": -0.30035489797592163,
|
||
|
|
"loss": 24.48188018798828,
|
||
|
|
"loss/core": 24.470382690429688,
|
||
|
|
"loss/preference_sft": 0.3016170859336853,
|
||
|
|
"loss/reference_anchor": 0.1231255754828453,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8962314128875732,
|
||
|
|
"rewards/margins": 0.5943207144737244,
|
||
|
|
"rewards/rejected": 1.3019107580184937,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3444829285144806,
|
||
|
|
"drift/rejected_abs": 0.13935281336307526,
|
||
|
|
"epoch": 0.7500298578765078,
|
||
|
|
"grad_norm": 42.5,
|
||
|
|
"learning_rate": 3.134507321515106e-08,
|
||
|
|
"logits/chosen": -1.9970579147338867,
|
||
|
|
"logits/rejected": -1.953643560409546,
|
||
|
|
"logps/chosen": -0.30979782342910767,
|
||
|
|
"logps/rejected": -0.2892361879348755,
|
||
|
|
"loss": 24.037694931030273,
|
||
|
|
"loss/core": 23.987783432006836,
|
||
|
|
"loss/preference_sft": 0.30979782342910767,
|
||
|
|
"loss/reference_anchor": 0.6345094442367554,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.4429678916931152,
|
||
|
|
"rewards/margins": 2.0524790287017822,
|
||
|
|
"rewards/rejected": 1.390488862991333,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26237040758132935,
|
||
|
|
"drift/rejected_abs": 0.19609911739826202,
|
||
|
|
"epoch": 0.7548071181177595,
|
||
|
|
"grad_norm": 780.0,
|
||
|
|
"learning_rate": 2.875309063423956e-08,
|
||
|
|
"logits/chosen": -2.1686906814575195,
|
||
|
|
"logits/rejected": -1.9951951503753662,
|
||
|
|
"logps/chosen": -0.28583893179893494,
|
||
|
|
"logps/rejected": -0.315361887216568,
|
||
|
|
"loss": 24.721479415893555,
|
||
|
|
"loss/core": 24.67513656616211,
|
||
|
|
"loss/preference_sft": 0.28583893179893494,
|
||
|
|
"loss/reference_anchor": 0.5893004536628723,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.6218254566192627,
|
||
|
|
"rewards/margins": 0.6629058122634888,
|
||
|
|
"rewards/rejected": 1.9589197635650635,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24005870521068573,
|
||
|
|
"drift/rejected_abs": 0.11813025176525116,
|
||
|
|
"epoch": 0.7595843783590112,
|
||
|
|
"grad_norm": 140.0,
|
||
|
|
"learning_rate": 2.626640959949375e-08,
|
||
|
|
"logits/chosen": -1.971401572227478,
|
||
|
|
"logits/rejected": -1.991015076637268,
|
||
|
|
"logps/chosen": -0.30549412965774536,
|
||
|
|
"logps/rejected": -0.296720415353775,
|
||
|
|
"loss": 24.04024887084961,
|
||
|
|
"loss/core": 23.997535705566406,
|
||
|
|
"loss/preference_sft": 0.30549412965774536,
|
||
|
|
"loss/reference_anchor": 0.5389922857284546,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.399735927581787,
|
||
|
|
"rewards/margins": 1.2197308540344238,
|
||
|
|
"rewards/rejected": 1.1800049543380737,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2291211634874344,
|
||
|
|
"drift/rejected_abs": 0.12003247439861298,
|
||
|
|
"epoch": 0.7643616386002627,
|
||
|
|
"grad_norm": 59.75,
|
||
|
|
"learning_rate": 2.388621363640797e-08,
|
||
|
|
"logits/chosen": -2.1559276580810547,
|
||
|
|
"logits/rejected": -2.1329169273376465,
|
||
|
|
"logps/chosen": -0.28992101550102234,
|
||
|
|
"logps/rejected": -0.2854466736316681,
|
||
|
|
"loss": 24.079273223876953,
|
||
|
|
"loss/core": 24.065502166748047,
|
||
|
|
"loss/preference_sft": 0.28992101550102234,
|
||
|
|
"loss/reference_anchor": 0.15460558235645294,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.290127754211426,
|
||
|
|
"rewards/margins": 1.0904570817947388,
|
||
|
|
"rewards/rejected": 1.1996705532073975,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22606876492500305,
|
||
|
|
"drift/rejected_abs": 0.2016010731458664,
|
||
|
|
"epoch": 0.7691388988415144,
|
||
|
|
"grad_norm": 104.0,
|
||
|
|
"learning_rate": 2.1613635589349756e-08,
|
||
|
|
"logits/chosen": -2.028775453567505,
|
||
|
|
"logits/rejected": -2.0167717933654785,
|
||
|
|
"logps/chosen": -0.2745590806007385,
|
||
|
|
"logps/rejected": -0.2676575779914856,
|
||
|
|
"loss": 24.971622467041016,
|
||
|
|
"loss/core": 24.949968338012695,
|
||
|
|
"loss/preference_sft": 0.2745590806007385,
|
||
|
|
"loss/reference_anchor": 0.2612423002719879,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.2601115703582764,
|
||
|
|
"rewards/margins": 0.24478550255298615,
|
||
|
|
"rewards/rejected": 2.0153260231018066,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15367405116558075,
|
||
|
|
"drift/rejected_abs": 0.14802715182304382,
|
||
|
|
"epoch": 0.773916159082766,
|
||
|
|
"grad_norm": 51.5,
|
||
|
|
"learning_rate": 1.944975708238708e-08,
|
||
|
|
"logits/chosen": -2.2288622856140137,
|
||
|
|
"logits/rejected": -2.1330811977386475,
|
||
|
|
"logps/chosen": -0.30788537859916687,
|
||
|
|
"logps/rejected": -0.31650078296661377,
|
||
|
|
"loss": 25.090599060058594,
|
||
|
|
"loss/core": 25.078384399414062,
|
||
|
|
"loss/preference_sft": 0.30788537859916687,
|
||
|
|
"loss/reference_anchor": 0.13205906748771667,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.534415364265442,
|
||
|
|
"rewards/margins": 0.05462510138750076,
|
||
|
|
"rewards/rejected": 1.4797903299331665,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2130316197872162,
|
||
|
|
"drift/rejected_abs": 0.11016812175512314,
|
||
|
|
"epoch": 0.7786934193240177,
|
||
|
|
"grad_norm": 145.0,
|
||
|
|
"learning_rate": 1.7395608004493884e-08,
|
||
|
|
"logits/chosen": -2.0962109565734863,
|
||
|
|
"logits/rejected": -2.1853909492492676,
|
||
|
|
"logps/chosen": -0.2830978333950043,
|
||
|
|
"logps/rejected": -0.2838820815086365,
|
||
|
|
"loss": 24.020706176757812,
|
||
|
|
"loss/core": 24.00937843322754,
|
||
|
|
"loss/preference_sft": 0.2830978333950043,
|
||
|
|
"loss/reference_anchor": 0.12274503707885742,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1303162574768066,
|
||
|
|
"rewards/margins": 1.032294750213623,
|
||
|
|
"rewards/rejected": 1.0980216264724731,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1970573216676712,
|
||
|
|
"drift/rejected_abs": 0.07308342307806015,
|
||
|
|
"epoch": 0.7834706795652693,
|
||
|
|
"grad_norm": 45.25,
|
||
|
|
"learning_rate": 1.5452166019378987e-08,
|
||
|
|
"logits/chosen": -2.099004030227661,
|
||
|
|
"logits/rejected": -2.1342720985412598,
|
||
|
|
"logps/chosen": -0.2875714898109436,
|
||
|
|
"logps/rejected": -0.31725940108299255,
|
||
|
|
"loss": 23.889062881469727,
|
||
|
|
"loss/core": 23.877830505371094,
|
||
|
|
"loss/preference_sft": 0.2875714898109436,
|
||
|
|
"loss/reference_anchor": 0.12097527086734772,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.9672977924346924,
|
||
|
|
"rewards/margins": 1.2818208932876587,
|
||
|
|
"rewards/rejected": 0.685477077960968,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2467678040266037,
|
||
|
|
"drift/rejected_abs": 0.12167330831289291,
|
||
|
|
"epoch": 0.788247939806521,
|
||
|
|
"grad_norm": 183.0,
|
||
|
|
"learning_rate": 1.3620356100170788e-08,
|
||
|
|
"logits/chosen": -2.1444103717803955,
|
||
|
|
"logits/rejected": -2.247798442840576,
|
||
|
|
"logps/chosen": -0.3017248809337616,
|
||
|
|
"logps/rejected": -0.28473973274230957,
|
||
|
|
"loss": 23.85074234008789,
|
||
|
|
"loss/core": 23.836971282958984,
|
||
|
|
"loss/preference_sft": 0.3017248809337616,
|
||
|
|
"loss/reference_anchor": 0.15346962213516235,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.466996669769287,
|
||
|
|
"rewards/margins": 1.2521899938583374,
|
||
|
|
"rewards/rejected": 1.2148066759109497,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17280618846416473,
|
||
|
|
"drift/rejected_abs": 0.16574548184871674,
|
||
|
|
"epoch": 0.7930252000477725,
|
||
|
|
"grad_norm": 91.0,
|
||
|
|
"learning_rate": 1.190105008918041e-08,
|
||
|
|
"logits/chosen": -2.1808249950408936,
|
||
|
|
"logits/rejected": -2.1297428607940674,
|
||
|
|
"logps/chosen": -0.2784227430820465,
|
||
|
|
"logps/rejected": -0.30759522318840027,
|
||
|
|
"loss": 25.10189437866211,
|
||
|
|
"loss/core": 25.08694839477539,
|
||
|
|
"loss/preference_sft": 0.2784227430820465,
|
||
|
|
"loss/reference_anchor": 0.17144832015037537,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7274526357650757,
|
||
|
|
"rewards/margins": 0.07077233493328094,
|
||
|
|
"rewards/rejected": 1.6566804647445679,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24050016701221466,
|
||
|
|
"drift/rejected_abs": 0.11438360065221786,
|
||
|
|
"epoch": 0.7978024602890242,
|
||
|
|
"grad_norm": 58.75,
|
||
|
|
"learning_rate": 1.0295066282951737e-08,
|
||
|
|
"logits/chosen": -2.096271276473999,
|
||
|
|
"logits/rejected": -2.096055507659912,
|
||
|
|
"logps/chosen": -0.2547900378704071,
|
||
|
|
"logps/rejected": -0.3017795979976654,
|
||
|
|
"loss": 23.816970825195312,
|
||
|
|
"loss/core": 23.80051040649414,
|
||
|
|
"loss/preference_sft": 0.2547900378704071,
|
||
|
|
"loss/reference_anchor": 0.19398611783981323,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.4034855365753174,
|
||
|
|
"rewards/margins": 1.5796620845794678,
|
||
|
|
"rewards/rejected": 0.8238231539726257,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28091713786125183,
|
||
|
|
"drift/rejected_abs": 0.10494539886713028,
|
||
|
|
"epoch": 0.8025797205302759,
|
||
|
|
"grad_norm": 796.0,
|
||
|
|
"learning_rate": 8.803169042796765e-09,
|
||
|
|
"logits/chosen": -2.0359435081481934,
|
||
|
|
"logits/rejected": -2.085045576095581,
|
||
|
|
"logps/chosen": -0.3219825327396393,
|
||
|
|
"logps/rejected": -0.3002759516239166,
|
||
|
|
"loss": 23.931066513061523,
|
||
|
|
"loss/core": 23.892078399658203,
|
||
|
|
"loss/preference_sft": 0.3219825327396393,
|
||
|
|
"loss/reference_anchor": 0.4876865744590759,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8080499172210693,
|
||
|
|
"rewards/margins": 1.7595913410186768,
|
||
|
|
"rewards/rejected": 1.0484585762023926,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.141829252243042,
|
||
|
|
"drift/rejected_abs": 0.06329073011875153,
|
||
|
|
"epoch": 0.8073569807715275,
|
||
|
|
"grad_norm": 54.5,
|
||
|
|
"learning_rate": 7.4260684310008815e-09,
|
||
|
|
"logits/chosen": -2.1739792823791504,
|
||
|
|
"logits/rejected": -2.248871326446533,
|
||
|
|
"logps/chosen": -0.29366642236709595,
|
||
|
|
"logps/rejected": -0.3113446831703186,
|
||
|
|
"loss": 24.187685012817383,
|
||
|
|
"loss/core": 24.17953109741211,
|
||
|
|
"loss/preference_sft": 0.29366642236709595,
|
||
|
|
"loss/reference_anchor": 0.07929544895887375,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.4174824953079224,
|
||
|
|
"rewards/margins": 0.963509202003479,
|
||
|
|
"rewards/rejected": 0.45397335290908813,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15600529313087463,
|
||
|
|
"drift/rejected_abs": 0.09067943692207336,
|
||
|
|
"epoch": 0.8121342410127792,
|
||
|
|
"grad_norm": 314.0,
|
||
|
|
"learning_rate": 6.164419872871835e-09,
|
||
|
|
"logits/chosen": -2.2953596115112305,
|
||
|
|
"logits/rejected": -2.3431358337402344,
|
||
|
|
"logps/chosen": -0.2520250082015991,
|
||
|
|
"logps/rejected": -0.24778322875499725,
|
||
|
|
"loss": 24.37624168395996,
|
||
|
|
"loss/core": 24.37070083618164,
|
||
|
|
"loss/preference_sft": 0.2520250082015991,
|
||
|
|
"loss/reference_anchor": 0.04869046062231064,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.5578166246414185,
|
||
|
|
"rewards/margins": 0.6599420309066772,
|
||
|
|
"rewards/rejected": 0.8978745341300964,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27003389596939087,
|
||
|
|
"drift/rejected_abs": 0.12391098588705063,
|
||
|
|
"epoch": 0.8169115012540308,
|
||
|
|
"grad_norm": 143.0,
|
||
|
|
"learning_rate": 5.018823844792602e-09,
|
||
|
|
"logits/chosen": -2.0034213066101074,
|
||
|
|
"logits/rejected": -2.040942668914795,
|
||
|
|
"logps/chosen": -0.2998163104057312,
|
||
|
|
"logps/rejected": -0.2942708730697632,
|
||
|
|
"loss": 23.890283584594727,
|
||
|
|
"loss/core": 23.85587501525879,
|
||
|
|
"loss/preference_sft": 0.2998163104057312,
|
||
|
|
"loss/reference_anchor": 0.42880433797836304,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.6989176273345947,
|
||
|
|
"rewards/margins": 1.4601688385009766,
|
||
|
|
"rewards/rejected": 1.2387486696243286,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.409585177898407,
|
||
|
|
"drift/rejected_abs": 0.19676879048347473,
|
||
|
|
"epoch": 0.8216887614952825,
|
||
|
|
"grad_norm": 37.25,
|
||
|
|
"learning_rate": 3.989825588427281e-09,
|
||
|
|
"logits/chosen": -1.9499990940093994,
|
||
|
|
"logits/rejected": -2.038193702697754,
|
||
|
|
"logps/chosen": -0.2986469268798828,
|
||
|
|
"logps/rejected": -0.2962108850479126,
|
||
|
|
"loss": 23.53319549560547,
|
||
|
|
"loss/core": 23.46843910217285,
|
||
|
|
"loss/preference_sft": 0.2986469268798828,
|
||
|
|
"loss/reference_anchor": 0.8335119485855103,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.0953145027160645,
|
||
|
|
"rewards/margins": 2.138956308364868,
|
||
|
|
"rewards/rejected": 1.956358551979065,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24526743590831757,
|
||
|
|
"drift/rejected_abs": 0.15863600373268127,
|
||
|
|
"epoch": 0.826466021736534,
|
||
|
|
"grad_norm": 85.5,
|
||
|
|
"learning_rate": 3.077914851215585e-09,
|
||
|
|
"logits/chosen": -1.95840585231781,
|
||
|
|
"logits/rejected": -2.0312142372131348,
|
||
|
|
"logps/chosen": -0.29089808464050293,
|
||
|
|
"logps/rejected": -0.29388031363487244,
|
||
|
|
"loss": 24.42396354675293,
|
||
|
|
"loss/core": 24.399206161499023,
|
||
|
|
"loss/preference_sft": 0.29089808464050293,
|
||
|
|
"loss/reference_anchor": 0.30098140239715576,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.4520413875579834,
|
||
|
|
"rewards/margins": 0.8663004040718079,
|
||
|
|
"rewards/rejected": 1.5857408046722412,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3167889714241028,
|
||
|
|
"drift/rejected_abs": 0.09579071402549744,
|
||
|
|
"epoch": 0.8312432819777857,
|
||
|
|
"grad_norm": 91.0,
|
||
|
|
"learning_rate": 2.2835256532794387e-09,
|
||
|
|
"logits/chosen": -2.1105716228485107,
|
||
|
|
"logits/rejected": -2.119257926940918,
|
||
|
|
"logps/chosen": -0.3130267262458801,
|
||
|
|
"logps/rejected": -0.2952987551689148,
|
||
|
|
"loss": 23.687124252319336,
|
||
|
|
"loss/core": 23.64649772644043,
|
||
|
|
"loss/preference_sft": 0.3130267262458801,
|
||
|
|
"loss/reference_anchor": 0.5103899240493774,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.165898561477661,
|
||
|
|
"rewards/margins": 2.2113327980041504,
|
||
|
|
"rewards/rejected": 0.9545663595199585,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1863785833120346,
|
||
|
|
"drift/rejected_abs": 0.10340656340122223,
|
||
|
|
"epoch": 0.8360205422190374,
|
||
|
|
"grad_norm": 66.0,
|
||
|
|
"learning_rate": 1.6070360808531359e-09,
|
||
|
|
"logits/chosen": -2.1814465522766113,
|
||
|
|
"logits/rejected": -2.22865891456604,
|
||
|
|
"logps/chosen": -0.3077881932258606,
|
||
|
|
"logps/rejected": -0.3306838274002075,
|
||
|
|
"loss": 24.516956329345703,
|
||
|
|
"loss/core": 24.487125396728516,
|
||
|
|
"loss/preference_sft": 0.3077881932258606,
|
||
|
|
"loss/reference_anchor": 0.36692848801612854,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.8635761737823486,
|
||
|
|
"rewards/margins": 0.8385453224182129,
|
||
|
|
"rewards/rejected": 1.0250307321548462,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24594731628894806,
|
||
|
|
"drift/rejected_abs": 0.18863597512245178,
|
||
|
|
"epoch": 0.840797802460289,
|
||
|
|
"grad_norm": 194.0,
|
||
|
|
"learning_rate": 1.0487681063345854e-09,
|
||
|
|
"logits/chosen": -2.1177544593811035,
|
||
|
|
"logits/rejected": -2.1003167629241943,
|
||
|
|
"logps/chosen": -0.2906046509742737,
|
||
|
|
"logps/rejected": -0.3307008743286133,
|
||
|
|
"loss": 25.163158416748047,
|
||
|
|
"loss/core": 25.12496566772461,
|
||
|
|
"loss/preference_sft": 0.2906046509742737,
|
||
|
|
"loss/reference_anchor": 0.4801712930202484,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.458425521850586,
|
||
|
|
"rewards/margins": 0.6850012540817261,
|
||
|
|
"rewards/rejected": 1.7734243869781494,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27374953031539917,
|
||
|
|
"drift/rejected_abs": 0.11317259073257446,
|
||
|
|
"epoch": 0.8455750627015407,
|
||
|
|
"grad_norm": 96.0,
|
||
|
|
"learning_rate": 6.089874350439505e-10,
|
||
|
|
"logits/chosen": -2.0427865982055664,
|
||
|
|
"logits/rejected": -2.1189193725585938,
|
||
|
|
"logps/chosen": -0.2813889980316162,
|
||
|
|
"logps/rejected": -0.2768685817718506,
|
||
|
|
"loss": 23.570148468017578,
|
||
|
|
"loss/core": 23.551849365234375,
|
||
|
|
"loss/preference_sft": 0.2813889980316162,
|
||
|
|
"loss/reference_anchor": 0.215844064950943,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.7366836071014404,
|
||
|
|
"rewards/margins": 1.611907958984375,
|
||
|
|
"rewards/rejected": 1.1247756481170654,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.246209055185318,
|
||
|
|
"drift/rejected_abs": 0.1439971625804901,
|
||
|
|
"epoch": 0.8503523229427923,
|
||
|
|
"grad_norm": 130.0,
|
||
|
|
"learning_rate": 2.8790337876233305e-10,
|
||
|
|
"logits/chosen": -1.9758459329605103,
|
||
|
|
"logits/rejected": -2.097522735595703,
|
||
|
|
"logps/chosen": -0.29943814873695374,
|
||
|
|
"logps/rejected": -0.27435192465782166,
|
||
|
|
"loss": 24.266326904296875,
|
||
|
|
"loss/core": 24.23988914489746,
|
||
|
|
"loss/preference_sft": 0.29943814873695374,
|
||
|
|
"loss/reference_anchor": 0.3225429058074951,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.461657762527466,
|
||
|
|
"rewards/margins": 1.0233776569366455,
|
||
|
|
"rewards/rejected": 1.4382802248001099,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23041865229606628,
|
||
|
|
"drift/rejected_abs": 0.15222403407096863,
|
||
|
|
"epoch": 0.855129583184044,
|
||
|
|
"grad_norm": 172.0,
|
||
|
|
"learning_rate": 8.566875611068503e-11,
|
||
|
|
"logits/chosen": -2.044543504714966,
|
||
|
|
"logits/rejected": -2.07798433303833,
|
||
|
|
"logps/chosen": -0.27221474051475525,
|
||
|
|
"logps/rejected": -0.2905210852622986,
|
||
|
|
"loss": 24.323848724365234,
|
||
|
|
"loss/core": 24.309627532958984,
|
||
|
|
"loss/preference_sft": 0.27221474051475525,
|
||
|
|
"loss/reference_anchor": 0.162424698472023,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.3031530380249023,
|
||
|
|
"rewards/margins": 0.7896323800086975,
|
||
|
|
"rewards/rejected": 1.51352059841156,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31285884976387024,
|
||
|
|
"drift/rejected_abs": 0.08625397086143494,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"grad_norm": 135.0,
|
||
|
|
"learning_rate": 2.3798198163782478e-12,
|
||
|
|
"logits/chosen": -2.055809736251831,
|
||
|
|
"logits/rejected": -2.176356315612793,
|
||
|
|
"logps/chosen": -0.2678307294845581,
|
||
|
|
"logps/rejected": -0.268280953168869,
|
||
|
|
"loss": 23.679527282714844,
|
||
|
|
"loss/core": 23.63846778869629,
|
||
|
|
"loss/preference_sft": 0.2678307294845581,
|
||
|
|
"loss/reference_anchor": 0.5206762552261353,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 3.1283411979675293,
|
||
|
|
"rewards/margins": 2.278930187225342,
|
||
|
|
"rewards/rejected": 0.8494111895561218,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"step": 900,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 24.184869588216145,
|
||
|
|
"train_runtime": 7109.0777,
|
||
|
|
"train_samples_per_second": 2.026,
|
||
|
|
"train_steps_per_second": 0.127
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 900,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 900,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|