76f5016a0e
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed0 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 19.311729431152344,
|
|
"kl": 7.840065002441406,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36431578.50708661,
|
|
"logits/rejected": -37226997.680620156,
|
|
"logps/chosen": -466.71574803149605,
|
|
"logps/rejected": -375.28842054263566,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.9136810302734375,
|
|
"metrics/advantage_var": 209.6494598388672,
|
|
"regularization/lipschitz_norm": 879.8392944335938,
|
|
"regularization/mmd": 0.1262146532535553,
|
|
"regularization/rkhs_norm": 150.2555389404297,
|
|
"regularization/w1": 0.7390649914741516,
|
|
"rewards/chosen": 0.08830630084661048,
|
|
"rewards/margins": 0.07595446117484714,
|
|
"rewards/rejected": 0.01235183967176334,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 18.068716049194336,
|
|
"kl": 12.981042861938477,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36214288.56702619,
|
|
"logits/rejected": -37964902.23771791,
|
|
"logps/chosen": -483.87788906009246,
|
|
"logps/rejected": -375.7964045166403,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.8673017024993896,
|
|
"metrics/advantage_var": 193.134033203125,
|
|
"regularization/lipschitz_norm": 896.8826293945312,
|
|
"regularization/mmd": 0.1289929896593094,
|
|
"regularization/rkhs_norm": 153.56309509277344,
|
|
"regularization/w1": 0.7533814311027527,
|
|
"rewards/chosen": 0.19668764840289146,
|
|
"rewards/margins": 0.10986243026266265,
|
|
"rewards/rejected": 0.08682521814022881,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 25.2999210357666,
|
|
"kl": 7.072659492492676,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35756625.76124031,
|
|
"logits/rejected": -36375034.35590551,
|
|
"logps/chosen": -474.3420058139535,
|
|
"logps/rejected": -358.6579232283465,
|
|
"loss": 0.5898,
|
|
"loss/base_kto": 3.890444278717041,
|
|
"metrics/advantage_var": 160.73265075683594,
|
|
"regularization/lipschitz_norm": 842.654296875,
|
|
"regularization/mmd": 0.12045440822839737,
|
|
"regularization/rkhs_norm": 143.3981170654297,
|
|
"regularization/w1": 0.7078295946121216,
|
|
"rewards/chosen": 0.10429767933926841,
|
|
"rewards/margins": 0.11426339272009703,
|
|
"rewards/rejected": -0.009965713380828618,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 22.47340965270996,
|
|
"kl": 17.255449295043945,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -38147992.50229709,
|
|
"logits/rejected": -39060184.39553429,
|
|
"logps/chosen": -490.06527565084224,
|
|
"logps/rejected": -375.5648425039872,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.882554292678833,
|
|
"metrics/advantage_var": 186.92015075683594,
|
|
"regularization/lipschitz_norm": 928.38232421875,
|
|
"regularization/mmd": 0.1312989443540573,
|
|
"regularization/rkhs_norm": 156.3082733154297,
|
|
"regularization/w1": 0.7798411250114441,
|
|
"rewards/chosen": 0.25746741302163095,
|
|
"rewards/margins": 0.09680484865728436,
|
|
"rewards/rejected": 0.1606625643643466,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 21.319414138793945,
|
|
"kl": 6.023135185241699,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36376466.408026755,
|
|
"logits/rejected": -37470520.30498534,
|
|
"logps/chosen": -508.5541387959866,
|
|
"logps/rejected": -383.38899835043986,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.8355255126953125,
|
|
"metrics/advantage_var": 183.87266540527344,
|
|
"regularization/lipschitz_norm": 915.0607299804688,
|
|
"regularization/mmd": 0.13171826303005219,
|
|
"regularization/rkhs_norm": 156.80743408203125,
|
|
"regularization/w1": 0.7686509490013123,
|
|
"rewards/chosen": -0.0059871546002136025,
|
|
"rewards/margins": 0.15239483607304177,
|
|
"rewards/rejected": -0.15838199067325537,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 17.00433349609375,
|
|
"kl": 10.309918403625488,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35338938.69170579,
|
|
"logits/rejected": -37091644.30577223,
|
|
"logps/chosen": -455.42556729264476,
|
|
"logps/rejected": -358.82958755850234,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.870764970779419,
|
|
"metrics/advantage_var": 211.3104705810547,
|
|
"regularization/lipschitz_norm": 941.3049926757812,
|
|
"regularization/mmd": 0.1367132067680359,
|
|
"regularization/rkhs_norm": 162.7538299560547,
|
|
"regularization/w1": 0.7906962633132935,
|
|
"rewards/chosen": 0.07709573617377005,
|
|
"rewards/margins": 0.11271761728680754,
|
|
"rewards/rejected": -0.03562188111303749,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 16.512834548950195,
|
|
"kl": 8.221867561340332,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36854272.0,
|
|
"logits/rejected": -37458937.559748426,
|
|
"logps/chosen": -499.73282220496895,
|
|
"logps/rejected": -380.55640723270443,
|
|
"loss": 0.6038,
|
|
"loss/base_kto": 3.8781769275665283,
|
|
"metrics/advantage_var": 206.3916015625,
|
|
"regularization/lipschitz_norm": 970.337890625,
|
|
"regularization/mmd": 0.13737133145332336,
|
|
"regularization/rkhs_norm": 163.53729248046875,
|
|
"regularization/w1": 0.8150838017463684,
|
|
"rewards/chosen": 0.08422803582612032,
|
|
"rewards/margins": 0.11420032516942043,
|
|
"rewards/rejected": -0.029972289343300106,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 18.08220672607422,
|
|
"kl": 10.539210319519043,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35482458.03058104,
|
|
"logits/rejected": -36236935.76996805,
|
|
"logps/chosen": -473.017249617737,
|
|
"logps/rejected": -363.83384085463257,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.817993640899658,
|
|
"metrics/advantage_var": 181.1797332763672,
|
|
"regularization/lipschitz_norm": 921.32861328125,
|
|
"regularization/mmd": 0.13110749423503876,
|
|
"regularization/rkhs_norm": 156.0803680419922,
|
|
"regularization/w1": 0.7739160656929016,
|
|
"rewards/chosen": 0.16040916326213686,
|
|
"rewards/margins": 0.14777508777437237,
|
|
"rewards/rejected": 0.012634075487764499,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.726795196533203,
|
|
"kl": 16.892858505249023,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36463163.6809816,
|
|
"logits/rejected": -36880351.38853503,
|
|
"logps/chosen": -460.85649923312883,
|
|
"logps/rejected": -363.66182324840764,
|
|
"loss": 0.5967,
|
|
"loss/base_kto": 3.7915825843811035,
|
|
"metrics/advantage_var": 213.2284698486328,
|
|
"regularization/lipschitz_norm": 998.6449584960938,
|
|
"regularization/mmd": 0.1433025598526001,
|
|
"regularization/rkhs_norm": 170.5983123779297,
|
|
"regularization/w1": 0.8388616442680359,
|
|
"rewards/chosen": 0.3729892098830521,
|
|
"rewards/margins": 0.18400788032353158,
|
|
"rewards/rejected": 0.18898132955952054,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 24.64959716796875,
|
|
"kl": 4.646457672119141,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -38184963.15076923,
|
|
"logits/rejected": -37582861.0031746,
|
|
"logps/chosen": -493.7851923076923,
|
|
"logps/rejected": -403.224181547619,
|
|
"loss": 0.6094,
|
|
"loss/base_kto": 3.879265546798706,
|
|
"metrics/advantage_var": 216.05857849121094,
|
|
"regularization/lipschitz_norm": 1018.1420288085938,
|
|
"regularization/mmd": 0.14044605195522308,
|
|
"regularization/rkhs_norm": 167.19769287109375,
|
|
"regularization/w1": 0.85523921251297,
|
|
"rewards/chosen": -0.010728354087242714,
|
|
"rewards/margins": 0.12415415808103605,
|
|
"rewards/rejected": -0.13488251216827876,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 18.703136444091797,
|
|
"kl": 17.4063663482666,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37278605.86499215,
|
|
"logits/rejected": -37374381.984447904,
|
|
"logps/chosen": -491.4295035321821,
|
|
"logps/rejected": -377.1530909797823,
|
|
"loss": 0.5926,
|
|
"loss/base_kto": 3.804840087890625,
|
|
"metrics/advantage_var": 200.4795379638672,
|
|
"regularization/lipschitz_norm": 951.6018676757812,
|
|
"regularization/mmd": 0.1365826427936554,
|
|
"regularization/rkhs_norm": 162.59837341308594,
|
|
"regularization/w1": 0.7993456125259399,
|
|
"rewards/chosen": 0.30602907573035226,
|
|
"rewards/margins": 0.1744748681584238,
|
|
"rewards/rejected": 0.13155420757192846,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 18.86064338684082,
|
|
"kl": 8.311650276184082,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -38200714.35598706,
|
|
"logits/rejected": -38060530.07854985,
|
|
"logps/chosen": -480.3775283171521,
|
|
"logps/rejected": -380.73022092145015,
|
|
"loss": 0.6065,
|
|
"loss/base_kto": 3.8396377563476562,
|
|
"metrics/advantage_var": 243.49658203125,
|
|
"regularization/lipschitz_norm": 1030.6707763671875,
|
|
"regularization/mmd": 0.1466662734746933,
|
|
"regularization/rkhs_norm": 174.60272216796875,
|
|
"regularization/w1": 0.8657633066177368,
|
|
"rewards/chosen": 0.037998779691924556,
|
|
"rewards/margins": 0.13388583994376557,
|
|
"rewards/rejected": -0.09588706025184102,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 22.847320556640625,
|
|
"kl": 5.043769359588623,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36546416.5732899,
|
|
"logits/rejected": -38108802.69069069,
|
|
"logps/chosen": -517.4850875407166,
|
|
"logps/rejected": -372.63079485735733,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.7869293689727783,
|
|
"metrics/advantage_var": 246.16172790527344,
|
|
"regularization/lipschitz_norm": 1021.5092163085938,
|
|
"regularization/mmd": 0.14847134053707123,
|
|
"regularization/rkhs_norm": 176.75160217285156,
|
|
"regularization/w1": 0.8580676913261414,
|
|
"rewards/chosen": 0.005644556753798494,
|
|
"rewards/margins": 0.19668232761792434,
|
|
"rewards/rejected": -0.19103777086412585,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 18.246173858642578,
|
|
"kl": 1.4904935359954834,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37131983.2576,
|
|
"logits/rejected": -38426283.18778626,
|
|
"logps/chosen": -517.8604,
|
|
"logps/rejected": -392.2879770992366,
|
|
"loss": 0.5948,
|
|
"loss/base_kto": 3.756671905517578,
|
|
"metrics/advantage_var": 236.1463623046875,
|
|
"regularization/lipschitz_norm": 1015.0855712890625,
|
|
"regularization/mmd": 0.14928221702575684,
|
|
"regularization/rkhs_norm": 177.7169189453125,
|
|
"regularization/w1": 0.8526718020439148,
|
|
"rewards/chosen": -0.12592723388671875,
|
|
"rewards/margins": 0.23520362176094348,
|
|
"rewards/rejected": -0.36113085564766223,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 22.975313186645508,
|
|
"kl": 9.60496997833252,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36020672.0,
|
|
"logits/rejected": -36922768.0,
|
|
"logps/chosen": -497.00810546875,
|
|
"logps/rejected": -351.035595703125,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.7739624977111816,
|
|
"metrics/advantage_var": 226.35171508789062,
|
|
"regularization/lipschitz_norm": 996.1676635742188,
|
|
"regularization/mmd": 0.1456291526556015,
|
|
"regularization/rkhs_norm": 173.36805725097656,
|
|
"regularization/w1": 0.836780846118927,
|
|
"rewards/chosen": 0.0830356776714325,
|
|
"rewards/margins": 0.209895521402359,
|
|
"rewards/rejected": -0.1268598437309265,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 20.777921676635742,
|
|
"kl": 11.073346138000488,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35355735.681957185,
|
|
"logits/rejected": -38105650.70926517,
|
|
"logps/chosen": -488.3495317278287,
|
|
"logps/rejected": -383.5483226837061,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.771421432495117,
|
|
"metrics/advantage_var": 217.07473754882812,
|
|
"regularization/lipschitz_norm": 996.0587768554688,
|
|
"regularization/mmd": 0.14185383915901184,
|
|
"regularization/rkhs_norm": 168.8736114501953,
|
|
"regularization/w1": 0.8366894125938416,
|
|
"rewards/chosen": 0.199867271866638,
|
|
"rewards/margins": 0.20280633005468945,
|
|
"rewards/rejected": -0.0029390581880514616,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 22.98124885559082,
|
|
"kl": 14.724398612976074,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -38076933.48851455,
|
|
"logits/rejected": -38290397.703349285,
|
|
"logps/chosen": -491.70348392036755,
|
|
"logps/rejected": -378.34968102073367,
|
|
"loss": 0.6078,
|
|
"loss/base_kto": 3.835813522338867,
|
|
"metrics/advantage_var": 249.15823364257812,
|
|
"regularization/lipschitz_norm": 1045.0399169921875,
|
|
"regularization/mmd": 0.148451566696167,
|
|
"regularization/rkhs_norm": 176.7280731201172,
|
|
"regularization/w1": 0.8778334856033325,
|
|
"rewards/chosen": 0.21668377769668837,
|
|
"rewards/margins": 0.13880619153879986,
|
|
"rewards/rejected": 0.0778775861578885,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 18.941415786743164,
|
|
"kl": 5.043925762176514,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36666190.76923077,
|
|
"logits/rejected": -37651927.414634146,
|
|
"logps/chosen": -497.1409254807692,
|
|
"logps/rejected": -385.7756526295732,
|
|
"loss": 0.5933,
|
|
"loss/base_kto": 3.791677236557007,
|
|
"metrics/advantage_var": 206.26358032226562,
|
|
"regularization/lipschitz_norm": 972.7359619140625,
|
|
"regularization/mmd": 0.13735435903072357,
|
|
"regularization/rkhs_norm": 163.51708984375,
|
|
"regularization/w1": 0.8170982599258423,
|
|
"rewards/chosen": -0.049271247325799405,
|
|
"rewards/margins": 0.19350429756183635,
|
|
"rewards/rejected": -0.24277554488763575,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 21.00739860534668,
|
|
"kl": 8.852449417114258,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36170224.87091757,
|
|
"logits/rejected": -37332088.565149136,
|
|
"logps/chosen": -483.83048211508554,
|
|
"logps/rejected": -353.101942700157,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.735443115234375,
|
|
"metrics/advantage_var": 218.0341339111328,
|
|
"regularization/lipschitz_norm": 980.8146362304688,
|
|
"regularization/mmd": 0.14092817902565002,
|
|
"regularization/rkhs_norm": 167.77163696289062,
|
|
"regularization/w1": 0.8238841891288757,
|
|
"rewards/chosen": 0.18129745649512782,
|
|
"rewards/margins": 0.2546149438435077,
|
|
"rewards/rejected": -0.07331748734837985,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 20.023303985595703,
|
|
"kl": 8.194683074951172,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36072057.304615386,
|
|
"logits/rejected": -38544361.244444445,
|
|
"logps/chosen": -508.0848076923077,
|
|
"logps/rejected": -365.3905257936508,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7646138668060303,
|
|
"metrics/advantage_var": 239.51992797851562,
|
|
"regularization/lipschitz_norm": 1040.847900390625,
|
|
"regularization/mmd": 0.1498996913433075,
|
|
"regularization/rkhs_norm": 178.45201110839844,
|
|
"regularization/w1": 0.8743122220039368,
|
|
"rewards/chosen": 0.14487689678485577,
|
|
"rewards/margins": 0.22515206603631288,
|
|
"rewards/rejected": -0.0802751692514571,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 23.46453094482422,
|
|
"kl": 9.771958351135254,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37220949.333333336,
|
|
"logits/rejected": -37654748.35443038,
|
|
"logps/chosen": -511.12789351851853,
|
|
"logps/rejected": -389.73838014240505,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.769212484359741,
|
|
"metrics/advantage_var": 246.97378540039062,
|
|
"regularization/lipschitz_norm": 1036.302978515625,
|
|
"regularization/mmd": 0.15128260850906372,
|
|
"regularization/rkhs_norm": 180.0983428955078,
|
|
"regularization/w1": 0.8704944849014282,
|
|
"rewards/chosen": 0.1615737279256185,
|
|
"rewards/margins": 0.18870579844285668,
|
|
"rewards/rejected": -0.027132070517238184,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 25.6339168548584,
|
|
"kl": 11.140782356262207,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37574342.929936305,
|
|
"logits/rejected": -38438701.54601227,
|
|
"logps/chosen": -488.5663813694268,
|
|
"logps/rejected": -377.37749233128835,
|
|
"loss": 0.5931,
|
|
"loss/base_kto": 3.769780397415161,
|
|
"metrics/advantage_var": 224.9917449951172,
|
|
"regularization/lipschitz_norm": 992.4802856445312,
|
|
"regularization/mmd": 0.1417173594236374,
|
|
"regularization/rkhs_norm": 168.7111358642578,
|
|
"regularization/w1": 0.833683431148529,
|
|
"rewards/chosen": 0.10946545934980842,
|
|
"rewards/margins": 0.2039143446531419,
|
|
"rewards/rejected": -0.0944488853033335,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 19.508899688720703,
|
|
"kl": 9.35730266571045,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36216771.3894081,
|
|
"logits/rejected": -37098280.92789969,
|
|
"logps/chosen": -476.0102219626168,
|
|
"logps/rejected": -365.8579055642633,
|
|
"loss": 0.5869,
|
|
"loss/base_kto": 3.680236577987671,
|
|
"metrics/advantage_var": 237.09336853027344,
|
|
"regularization/lipschitz_norm": 1030.716796875,
|
|
"regularization/mmd": 0.148957759141922,
|
|
"regularization/rkhs_norm": 177.33067321777344,
|
|
"regularization/w1": 0.865801990032196,
|
|
"rewards/chosen": 0.15998747638452834,
|
|
"rewards/margins": 0.3211469080572973,
|
|
"rewards/rejected": -0.16115943167276892,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 21.36751365661621,
|
|
"kl": 8.980067253112793,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -35576658.11320755,
|
|
"logits/rejected": -36327990.0621118,
|
|
"logps/chosen": -473.73309748427675,
|
|
"logps/rejected": -381.9711277173913,
|
|
"loss": 0.6033,
|
|
"loss/base_kto": 3.8249454498291016,
|
|
"metrics/advantage_var": 228.97879028320312,
|
|
"regularization/lipschitz_norm": 1020.0426025390625,
|
|
"regularization/mmd": 0.14489972591400146,
|
|
"regularization/rkhs_norm": 172.49966430664062,
|
|
"regularization/w1": 0.8568357825279236,
|
|
"rewards/chosen": 0.011161331860524303,
|
|
"rewards/margins": 0.16597854578381865,
|
|
"rewards/rejected": -0.15481721392329434,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 20.054691314697266,
|
|
"kl": 16.56330108642578,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -34627515.10703364,
|
|
"logits/rejected": -34938219.143769965,
|
|
"logps/chosen": -470.923881880734,
|
|
"logps/rejected": -370.4764376996805,
|
|
"loss": 0.59,
|
|
"loss/base_kto": 3.7410390377044678,
|
|
"metrics/advantage_var": 230.6334991455078,
|
|
"regularization/lipschitz_norm": 991.7376098632812,
|
|
"regularization/mmd": 0.145973339676857,
|
|
"regularization/rkhs_norm": 173.7777862548828,
|
|
"regularization/w1": 0.8330594897270203,
|
|
"rewards/chosen": 0.3361468067227518,
|
|
"rewards/margins": 0.2523917340285157,
|
|
"rewards/rejected": 0.0837550726942361,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 20.063520431518555,
|
|
"kl": 6.0723557472229,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35328408.94041868,
|
|
"logits/rejected": -35994976.72837633,
|
|
"logps/chosen": -462.0980273752013,
|
|
"logps/rejected": -368.64501138088013,
|
|
"loss": 0.6028,
|
|
"loss/base_kto": 3.8176231384277344,
|
|
"metrics/advantage_var": 240.4467315673828,
|
|
"regularization/lipschitz_norm": 1020.6088256835938,
|
|
"regularization/mmd": 0.1475488394498825,
|
|
"regularization/rkhs_norm": 175.6533660888672,
|
|
"regularization/w1": 0.8573114275932312,
|
|
"rewards/chosen": -0.008806174887935128,
|
|
"rewards/margins": 0.16633828115637916,
|
|
"rewards/rejected": -0.1751444560443143,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 17.4051513671875,
|
|
"kl": 11.1605806350708,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -35956836.42199109,
|
|
"logits/rejected": -37862189.12685338,
|
|
"logps/chosen": -502.7159175334324,
|
|
"logps/rejected": -379.42202944810543,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.736018419265747,
|
|
"metrics/advantage_var": 275.1711120605469,
|
|
"regularization/lipschitz_norm": 1049.8695068359375,
|
|
"regularization/mmd": 0.15403233468532562,
|
|
"regularization/rkhs_norm": 183.371826171875,
|
|
"regularization/w1": 0.8818902969360352,
|
|
"rewards/chosen": 0.19694249236743128,
|
|
"rewards/margins": 0.2647455563021094,
|
|
"rewards/rejected": -0.06780306393467811,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 26.96074867248535,
|
|
"kl": 16.029020309448242,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36083062.4,
|
|
"logits/rejected": -37897600.0,
|
|
"logps/chosen": -485.854736328125,
|
|
"logps/rejected": -382.617138671875,
|
|
"loss": 0.5908,
|
|
"loss/base_kto": 3.724695920944214,
|
|
"metrics/advantage_var": 252.48672485351562,
|
|
"regularization/lipschitz_norm": 1010.6306762695312,
|
|
"regularization/mmd": 0.1526278257369995,
|
|
"regularization/rkhs_norm": 181.6997833251953,
|
|
"regularization/w1": 0.8489298224449158,
|
|
"rewards/chosen": 0.2972104072570801,
|
|
"rewards/margins": 0.23930780887603761,
|
|
"rewards/rejected": 0.05790259838104248,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 16.330476760864258,
|
|
"kl": 11.252641677856445,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35600269.158878505,
|
|
"logits/rejected": -37431767.87460815,
|
|
"logps/chosen": -473.065031152648,
|
|
"logps/rejected": -355.59037029780563,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.721024751663208,
|
|
"metrics/advantage_var": 290.3227233886719,
|
|
"regularization/lipschitz_norm": 1040.0142822265625,
|
|
"regularization/mmd": 0.15475240349769592,
|
|
"regularization/rkhs_norm": 184.2290496826172,
|
|
"regularization/w1": 0.8736120462417603,
|
|
"rewards/chosen": 0.15429973899389723,
|
|
"rewards/margins": 0.2587788680317842,
|
|
"rewards/rejected": -0.10447912903788695,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 20.427078247070312,
|
|
"kl": 15.229260444641113,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36289563.1575663,
|
|
"logits/rejected": -38472965.20813771,
|
|
"logps/chosen": -490.4938085023401,
|
|
"logps/rejected": -356.04279147104853,
|
|
"loss": 0.6032,
|
|
"loss/base_kto": 3.7195394039154053,
|
|
"metrics/advantage_var": 282.6444396972656,
|
|
"regularization/lipschitz_norm": 1126.998046875,
|
|
"regularization/mmd": 0.15921609103679657,
|
|
"regularization/rkhs_norm": 189.54296875,
|
|
"regularization/w1": 0.9466783404350281,
|
|
"rewards/chosen": 0.28862956459176337,
|
|
"rewards/margins": 0.24849377056258723,
|
|
"rewards/rejected": 0.04013579402917614,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 20.34117317199707,
|
|
"kl": 19.274032592773438,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37381039.282843895,
|
|
"logits/rejected": -37879128.56872038,
|
|
"logps/chosen": -492.3865919629057,
|
|
"logps/rejected": -362.41985090837284,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.7460694313049316,
|
|
"metrics/advantage_var": 224.95555114746094,
|
|
"regularization/lipschitz_norm": 1014.6459350585938,
|
|
"regularization/mmd": 0.14508013427257538,
|
|
"regularization/rkhs_norm": 172.7144317626953,
|
|
"regularization/w1": 0.8523025512695312,
|
|
"rewards/chosen": 0.26577787089753185,
|
|
"rewards/margins": 0.20545618356376347,
|
|
"rewards/rejected": 0.06032168733376839,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 17.052738189697266,
|
|
"kl": 14.50988483428955,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35958289.11854103,
|
|
"logits/rejected": -37832407.66559485,
|
|
"logps/chosen": -465.93669262917933,
|
|
"logps/rejected": -376.50015072347264,
|
|
"loss": 0.5994,
|
|
"loss/base_kto": 3.715284824371338,
|
|
"metrics/advantage_var": 262.3467712402344,
|
|
"regularization/lipschitz_norm": 1101.0782470703125,
|
|
"regularization/mmd": 0.15501636266708374,
|
|
"regularization/rkhs_norm": 184.5432891845703,
|
|
"regularization/w1": 0.9249057769775391,
|
|
"rewards/chosen": 0.2888654807418313,
|
|
"rewards/margins": 0.25439598970928917,
|
|
"rewards/rejected": 0.03446949103254214,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 23.290475845336914,
|
|
"kl": 6.941014289855957,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36664129.084745765,
|
|
"logits/rejected": -37686718.275752775,
|
|
"logps/chosen": -483.48237673343607,
|
|
"logps/rejected": -385.3207706022187,
|
|
"loss": 0.5941,
|
|
"loss/base_kto": 3.712000608444214,
|
|
"metrics/advantage_var": 261.04132080078125,
|
|
"regularization/lipschitz_norm": 1058.1522216796875,
|
|
"regularization/mmd": 0.15186677873134613,
|
|
"regularization/rkhs_norm": 180.79376220703125,
|
|
"regularization/w1": 0.8888479471206665,
|
|
"rewards/chosen": 0.11919425411841535,
|
|
"rewards/margins": 0.28848420728674934,
|
|
"rewards/rejected": -0.169289953168334,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 19.351591110229492,
|
|
"kl": 17.657398223876953,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35070033.01265823,
|
|
"logits/rejected": -36018748.04938272,
|
|
"logps/chosen": -471.8562598892405,
|
|
"logps/rejected": -369.0094521604938,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.7355103492736816,
|
|
"metrics/advantage_var": 231.151611328125,
|
|
"regularization/lipschitz_norm": 1005.7997436523438,
|
|
"regularization/mmd": 0.14701783657073975,
|
|
"regularization/rkhs_norm": 175.02122497558594,
|
|
"regularization/w1": 0.8448716998100281,
|
|
"rewards/chosen": 0.2693678940398784,
|
|
"rewards/margins": 0.21084230231910894,
|
|
"rewards/rejected": 0.05852559172076943,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 25.170379638671875,
|
|
"kl": 19.11185073852539,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35493323.03448276,
|
|
"logits/rejected": -36408256.19937695,
|
|
"logps/chosen": -465.24794278996865,
|
|
"logps/rejected": -353.55736468068534,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.796590566635132,
|
|
"metrics/advantage_var": 209.95127868652344,
|
|
"regularization/lipschitz_norm": 945.8595581054688,
|
|
"regularization/mmd": 0.13836148381233215,
|
|
"regularization/rkhs_norm": 164.71604919433594,
|
|
"regularization/w1": 0.7945219874382019,
|
|
"rewards/chosen": 0.22805635458249657,
|
|
"rewards/margins": 0.18373780782386268,
|
|
"rewards/rejected": 0.044318546758633906,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 18.53089141845703,
|
|
"kl": 7.533125400543213,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -37032651.99370079,
|
|
"logits/rejected": -37255634.75348837,
|
|
"logps/chosen": -494.34498031496065,
|
|
"logps/rejected": -368.0325096899225,
|
|
"loss": 0.603,
|
|
"loss/base_kto": 3.770843505859375,
|
|
"metrics/advantage_var": 255.7259063720703,
|
|
"regularization/lipschitz_norm": 1073.8582763671875,
|
|
"regularization/mmd": 0.15092627704143524,
|
|
"regularization/rkhs_norm": 179.67413330078125,
|
|
"regularization/w1": 0.902040958404541,
|
|
"rewards/chosen": 0.064196374848133,
|
|
"rewards/margins": 0.20598035676280113,
|
|
"rewards/rejected": -0.14178398191466812,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 22.40465545654297,
|
|
"kl": 15.472909927368164,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35584420.42923795,
|
|
"logits/rejected": -37623296.80376766,
|
|
"logps/chosen": -475.797433903577,
|
|
"logps/rejected": -387.8880003924647,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.7116806507110596,
|
|
"metrics/advantage_var": 265.4413757324219,
|
|
"regularization/lipschitz_norm": 1059.0657958984375,
|
|
"regularization/mmd": 0.15491236746311188,
|
|
"regularization/rkhs_norm": 184.4194793701172,
|
|
"regularization/w1": 0.8896151781082153,
|
|
"rewards/chosen": 0.22139997467468167,
|
|
"rewards/margins": 0.2652009784347156,
|
|
"rewards/rejected": -0.04380100376003391,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 15.138018608093262,
|
|
"kl": 12.29021167755127,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -34901480.01292407,
|
|
"logits/rejected": -35505169.040847205,
|
|
"logps/chosen": -476.82456583198706,
|
|
"logps/rejected": -361.4870461422088,
|
|
"loss": 0.589,
|
|
"loss/base_kto": 3.707359790802002,
|
|
"metrics/advantage_var": 245.03794860839844,
|
|
"regularization/lipschitz_norm": 1021.0166015625,
|
|
"regularization/mmd": 0.14694668352603912,
|
|
"regularization/rkhs_norm": 174.9365234375,
|
|
"regularization/w1": 0.8576539158821106,
|
|
"rewards/chosen": 0.176424510259428,
|
|
"rewards/margins": 0.27775879787111957,
|
|
"rewards/rejected": -0.10133428761169157,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 21.16314697265625,
|
|
"kl": 8.471266746520996,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36768361.2488408,
|
|
"logits/rejected": -37576100.310618065,
|
|
"logps/chosen": -505.9378863987635,
|
|
"logps/rejected": -376.27067650554676,
|
|
"loss": 0.6012,
|
|
"loss/base_kto": 3.5507023334503174,
|
|
"metrics/advantage_var": 387.4823913574219,
|
|
"regularization/lipschitz_norm": 1284.433349609375,
|
|
"regularization/mmd": 0.17976389825344086,
|
|
"regularization/rkhs_norm": 214.004638671875,
|
|
"regularization/w1": 1.0789238214492798,
|
|
"rewards/chosen": 0.18631688962579696,
|
|
"rewards/margins": 0.47616607572053743,
|
|
"rewards/rejected": -0.2898491860947405,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 22.055309295654297,
|
|
"kl": 15.419751167297363,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35734688.0,
|
|
"logits/rejected": -37085798.4,
|
|
"logps/chosen": -477.52314453125,
|
|
"logps/rejected": -373.0943359375,
|
|
"loss": 0.5918,
|
|
"loss/base_kto": 3.3308322429656982,
|
|
"metrics/advantage_var": 497.14727783203125,
|
|
"regularization/lipschitz_norm": 1427.1160888671875,
|
|
"regularization/mmd": 0.2049953043460846,
|
|
"regularization/rkhs_norm": 244.0420379638672,
|
|
"regularization/w1": 1.1987775564193726,
|
|
"rewards/chosen": 0.41237993240356446,
|
|
"rewards/margins": 0.717909860610962,
|
|
"rewards/rejected": -0.30552992820739744,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 21.182348251342773,
|
|
"kl": 19.323965072631836,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35165066.8410104,
|
|
"logits/rejected": -36809098.754530475,
|
|
"logps/chosen": -483.50204309063895,
|
|
"logps/rejected": -372.2024557248764,
|
|
"loss": 0.6083,
|
|
"loss/base_kto": 3.288318634033203,
|
|
"metrics/advantage_var": 644.7529296875,
|
|
"regularization/lipschitz_norm": 1602.8765869140625,
|
|
"regularization/mmd": 0.23192663490772247,
|
|
"regularization/rkhs_norm": 276.1031188964844,
|
|
"regularization/w1": 1.3464163541793823,
|
|
"rewards/chosen": 0.6143584796967868,
|
|
"rewards/margins": 0.7772543484915124,
|
|
"rewards/rejected": -0.1628958687947256,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 23.51076316833496,
|
|
"kl": 17.561798095703125,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36646173.164556965,
|
|
"logits/rejected": -36900064.39506173,
|
|
"logps/chosen": -472.94229628164555,
|
|
"logps/rejected": -363.1675829475309,
|
|
"loss": 0.6015,
|
|
"loss/base_kto": 3.3055858612060547,
|
|
"metrics/advantage_var": 532.6900024414062,
|
|
"regularization/lipschitz_norm": 1535.5950927734375,
|
|
"regularization/mmd": 0.21687713265419006,
|
|
"regularization/rkhs_norm": 258.18707275390625,
|
|
"regularization/w1": 1.2898999452590942,
|
|
"rewards/chosen": 0.6176550659952285,
|
|
"rewards/margins": 0.7515917007653745,
|
|
"rewards/rejected": -0.1339366347701461,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 16.954561233520508,
|
|
"kl": 9.474974632263184,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35014985.920972645,
|
|
"logits/rejected": -37162784.10289389,
|
|
"logps/chosen": -489.72193199088144,
|
|
"logps/rejected": -373.8094855305466,
|
|
"loss": 0.6223,
|
|
"loss/base_kto": 3.3072383403778076,
|
|
"metrics/advantage_var": 706.9900512695312,
|
|
"regularization/lipschitz_norm": 1706.0087890625,
|
|
"regularization/mmd": 0.2378632128238678,
|
|
"regularization/rkhs_norm": 283.1705017089844,
|
|
"regularization/w1": 1.4330474138259888,
|
|
"rewards/chosen": 0.46047593322568386,
|
|
"rewards/margins": 0.7952240572287108,
|
|
"rewards/rejected": -0.334748124003027,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 19.60972023010254,
|
|
"kl": 12.555362701416016,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34420921.89538462,
|
|
"logits/rejected": -36889746.28571428,
|
|
"logps/chosen": -491.0981730769231,
|
|
"logps/rejected": -360.1377976190476,
|
|
"loss": 0.6029,
|
|
"loss/base_kto": 3.2510335445404053,
|
|
"metrics/advantage_var": 604.8927612304688,
|
|
"regularization/lipschitz_norm": 1595.8828125,
|
|
"regularization/mmd": 0.23130464553833008,
|
|
"regularization/rkhs_norm": 275.3626708984375,
|
|
"regularization/w1": 1.3405417203903198,
|
|
"rewards/chosen": 0.5105926044170673,
|
|
"rewards/margins": 0.8058155135270004,
|
|
"rewards/rejected": -0.29522290910993304,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.558839797973633,
|
|
"kl": 26.411273956298828,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35239372.01230769,
|
|
"logits/rejected": -37002067.70793651,
|
|
"logps/chosen": -487.0657692307692,
|
|
"logps/rejected": -355.89489087301587,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.225663900375366,
|
|
"metrics/advantage_var": 574.6051635742188,
|
|
"regularization/lipschitz_norm": 1523.2562255859375,
|
|
"regularization/mmd": 0.22221890091896057,
|
|
"regularization/rkhs_norm": 264.5462951660156,
|
|
"regularization/w1": 1.2795352935791016,
|
|
"rewards/chosen": 0.6853428297776443,
|
|
"rewards/margins": 0.7736441341861264,
|
|
"rewards/rejected": -0.08830130440848215,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 20.727832794189453,
|
|
"kl": 20.686328887939453,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35798360.663414635,
|
|
"logits/rejected": -38131006.74887218,
|
|
"logps/chosen": -473.8318089430894,
|
|
"logps/rejected": -394.09807330827067,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.263370990753174,
|
|
"metrics/advantage_var": 560.9389038085938,
|
|
"regularization/lipschitz_norm": 1569.9447021484375,
|
|
"regularization/mmd": 0.21853144466876984,
|
|
"regularization/rkhs_norm": 260.156494140625,
|
|
"regularization/w1": 1.3187536001205444,
|
|
"rewards/chosen": 0.47452739932672766,
|
|
"rewards/margins": 0.7901420738303695,
|
|
"rewards/rejected": -0.3156146745036419,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 15.329206466674805,
|
|
"kl": 8.2859468460083,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35388444.96505824,
|
|
"logits/rejected": -36061470.53902798,
|
|
"logps/chosen": -492.9282445923461,
|
|
"logps/rejected": -387.2229381443299,
|
|
"loss": 0.6058,
|
|
"loss/base_kto": 3.2901668548583984,
|
|
"metrics/advantage_var": 576.5140991210938,
|
|
"regularization/lipschitz_norm": 1589.8365478515625,
|
|
"regularization/mmd": 0.22090554237365723,
|
|
"regularization/rkhs_norm": 262.9828186035156,
|
|
"regularization/w1": 1.3354628086090088,
|
|
"rewards/chosen": 0.3080719044918625,
|
|
"rewards/margins": 0.7662068660765688,
|
|
"rewards/rejected": -0.45813496158470635,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 18.086589813232422,
|
|
"kl": 4.244732856750488,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35996707.42138365,
|
|
"logits/rejected": -37513546.73291925,
|
|
"logps/chosen": -498.3534787735849,
|
|
"logps/rejected": -371.3962296195652,
|
|
"loss": 0.6034,
|
|
"loss/base_kto": 3.308680295944214,
|
|
"metrics/advantage_var": 571.4664916992188,
|
|
"regularization/lipschitz_norm": 1542.1009521484375,
|
|
"regularization/mmd": 0.22286613285541534,
|
|
"regularization/rkhs_norm": 265.31683349609375,
|
|
"regularization/w1": 1.2953647375106812,
|
|
"rewards/chosen": 0.11151643669080434,
|
|
"rewards/margins": 0.7771609224487624,
|
|
"rewards/rejected": -0.6656444857579581,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 16.93316078186035,
|
|
"kl": 6.464847087860107,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35316583.064935066,
|
|
"logits/rejected": -37417431.90361446,
|
|
"logps/chosen": -503.64960430194805,
|
|
"logps/rejected": -404.2452936746988,
|
|
"loss": 0.6098,
|
|
"loss/base_kto": 3.281092882156372,
|
|
"metrics/advantage_var": 610.0521850585938,
|
|
"regularization/lipschitz_norm": 1625.29296875,
|
|
"regularization/mmd": 0.23171770572662354,
|
|
"regularization/rkhs_norm": 275.8544006347656,
|
|
"regularization/w1": 1.3652461767196655,
|
|
"rewards/chosen": 0.2481203946200284,
|
|
"rewards/margins": 0.7917356209060318,
|
|
"rewards/rejected": -0.5436152262860033,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 22.382692337036133,
|
|
"kl": 7.046362400054932,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35062903.36196319,
|
|
"logits/rejected": -35767406.87898089,
|
|
"logps/chosen": -481.2234470858896,
|
|
"logps/rejected": -384.96775477707007,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.2337722778320312,
|
|
"metrics/advantage_var": 634.6586303710938,
|
|
"regularization/lipschitz_norm": 1645.6080322265625,
|
|
"regularization/mmd": 0.233300119638443,
|
|
"regularization/rkhs_norm": 277.7382507324219,
|
|
"regularization/w1": 1.3823107481002808,
|
|
"rewards/chosen": 0.42465172516056365,
|
|
"rewards/margins": 0.8859964501680277,
|
|
"rewards/rejected": -0.46134472500746415,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 17.184362411499023,
|
|
"kl": 13.68155574798584,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35164749.57575758,
|
|
"logits/rejected": -37453421.006451614,
|
|
"logps/chosen": -491.7760416666667,
|
|
"logps/rejected": -361.9875252016129,
|
|
"loss": 0.6174,
|
|
"loss/base_kto": 3.2585244178771973,
|
|
"metrics/advantage_var": 661.8715209960938,
|
|
"regularization/lipschitz_norm": 1715.8726806640625,
|
|
"regularization/mmd": 0.23919086158275604,
|
|
"regularization/rkhs_norm": 284.7510681152344,
|
|
"regularization/w1": 1.4413330554962158,
|
|
"rewards/chosen": 0.5577228892933238,
|
|
"rewards/margins": 0.7887860921820586,
|
|
"rewards/rejected": -0.2310632028887349,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 18.59825897216797,
|
|
"kl": 16.316736221313477,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35512621.643410854,
|
|
"logits/rejected": -37523468.9007874,
|
|
"logps/chosen": -490.4732558139535,
|
|
"logps/rejected": -368.9190452755906,
|
|
"loss": 0.6067,
|
|
"loss/base_kto": 3.2406487464904785,
|
|
"metrics/advantage_var": 648.6552124023438,
|
|
"regularization/lipschitz_norm": 1643.1043701171875,
|
|
"regularization/mmd": 0.2323632687330246,
|
|
"regularization/rkhs_norm": 276.6229553222656,
|
|
"regularization/w1": 1.3802077770233154,
|
|
"rewards/chosen": 0.5867794539577277,
|
|
"rewards/margins": 0.8252295839097454,
|
|
"rewards/rejected": -0.23845012995201773,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 18.296396255493164,
|
|
"kl": 10.483955383300781,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35389220.340694,
|
|
"logits/rejected": -37441843.51702786,
|
|
"logps/chosen": -491.8681979495268,
|
|
"logps/rejected": -354.929445626935,
|
|
"loss": 0.6143,
|
|
"loss/base_kto": 3.3042850494384766,
|
|
"metrics/advantage_var": 997.9747924804688,
|
|
"regularization/lipschitz_norm": 1640.1961669921875,
|
|
"regularization/mmd": 0.2319926768541336,
|
|
"regularization/rkhs_norm": 276.1817626953125,
|
|
"regularization/w1": 1.3777645826339722,
|
|
"rewards/chosen": 0.3760328804280856,
|
|
"rewards/margins": 0.79358496628093,
|
|
"rewards/rejected": -0.4175520858528444,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 16.5646915435791,
|
|
"kl": 11.669934272766113,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34477041.532182105,
|
|
"logits/rejected": -35190468.678071536,
|
|
"logps/chosen": -471.4804258241758,
|
|
"logps/rejected": -392.06118779160187,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.3053252696990967,
|
|
"metrics/advantage_var": 582.5371704101562,
|
|
"regularization/lipschitz_norm": 1547.6551513671875,
|
|
"regularization/mmd": 0.22402773797512054,
|
|
"regularization/rkhs_norm": 266.6996765136719,
|
|
"regularization/w1": 1.30003023147583,
|
|
"rewards/chosen": 0.42099458942982243,
|
|
"rewards/margins": 0.7436737679554712,
|
|
"rewards/rejected": -0.3226791785256488,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 16.393640518188477,
|
|
"kl": 11.862051963806152,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36113415.74583964,
|
|
"logits/rejected": -36087123.127625205,
|
|
"logps/chosen": -470.0517681543117,
|
|
"logps/rejected": -365.0215569466882,
|
|
"loss": 0.5972,
|
|
"loss/base_kto": 3.2723305225372314,
|
|
"metrics/advantage_var": 585.9185791015625,
|
|
"regularization/lipschitz_norm": 1528.84375,
|
|
"regularization/mmd": 0.22064733505249023,
|
|
"regularization/rkhs_norm": 262.6754150390625,
|
|
"regularization/w1": 1.2842286825180054,
|
|
"rewards/chosen": 0.4480481400251749,
|
|
"rewards/margins": 0.7883315107749982,
|
|
"rewards/rejected": -0.3402833707498233,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 24.36058235168457,
|
|
"kl": 15.893205642700195,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -36033401.30586371,
|
|
"logits/rejected": -37735154.19414484,
|
|
"logps/chosen": -477.97414817749603,
|
|
"logps/rejected": -400.28016660246533,
|
|
"loss": 0.6088,
|
|
"loss/base_kto": 3.276933431625366,
|
|
"metrics/advantage_var": 584.4525146484375,
|
|
"regularization/lipschitz_norm": 1632.7537841796875,
|
|
"regularization/mmd": 0.2217748612165451,
|
|
"regularization/rkhs_norm": 264.0176696777344,
|
|
"regularization/w1": 1.3715132474899292,
|
|
"rewards/chosen": 0.47072846764807846,
|
|
"rewards/margins": 0.7900672500070562,
|
|
"rewards/rejected": -0.3193387823589778,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.702255249023438,
|
|
"kl": 12.50069522857666,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -36280888.529968455,
|
|
"logits/rejected": -36241547.49226006,
|
|
"logps/chosen": -474.06259858044166,
|
|
"logps/rejected": -380.24934694272446,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.3043906688690186,
|
|
"metrics/advantage_var": 585.9420166015625,
|
|
"regularization/lipschitz_norm": 1509.2894287109375,
|
|
"regularization/mmd": 0.21785235404968262,
|
|
"regularization/rkhs_norm": 259.3480224609375,
|
|
"regularization/w1": 1.2678030729293823,
|
|
"rewards/chosen": 0.41566688706070093,
|
|
"rewards/margins": 0.7549185913209926,
|
|
"rewards/rejected": -0.3392517042602917,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 16.96848487854004,
|
|
"kl": 13.61374568939209,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34639646.91021672,
|
|
"logits/rejected": -36730149.95583596,
|
|
"logps/chosen": -505.5739164086687,
|
|
"logps/rejected": -360.9603213722398,
|
|
"loss": 0.6069,
|
|
"loss/base_kto": 3.2965340614318848,
|
|
"metrics/advantage_var": 588.7193603515625,
|
|
"regularization/lipschitz_norm": 1588.00439453125,
|
|
"regularization/mmd": 0.22466149926185608,
|
|
"regularization/rkhs_norm": 267.4541931152344,
|
|
"regularization/w1": 1.3339236974716187,
|
|
"rewards/chosen": 0.5111178466041022,
|
|
"rewards/margins": 0.7775391435992471,
|
|
"rewards/rejected": -0.26642129699514494,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 19.33449363708496,
|
|
"kl": 9.589579582214355,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -36402548.36363637,
|
|
"logits/rejected": -36827897.8313253,
|
|
"logps/chosen": -492.31929788961037,
|
|
"logps/rejected": -393.4038968373494,
|
|
"loss": 0.6016,
|
|
"loss/base_kto": 3.3590850830078125,
|
|
"metrics/advantage_var": 501.6669006347656,
|
|
"regularization/lipschitz_norm": 1483.150634765625,
|
|
"regularization/mmd": 0.2078084498643875,
|
|
"regularization/rkhs_norm": 247.3909912109375,
|
|
"regularization/w1": 1.2458465099334717,
|
|
"rewards/chosen": 0.30930903050806613,
|
|
"rewards/margins": 0.6699329991609981,
|
|
"rewards/rejected": -0.36062396865293206,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 22.132320404052734,
|
|
"kl": 6.338320255279541,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34132691.80456026,
|
|
"logits/rejected": -35852700.06006006,
|
|
"logps/chosen": -499.4040105863192,
|
|
"logps/rejected": -364.59030123873873,
|
|
"loss": 0.6197,
|
|
"loss/base_kto": 3.3146088123321533,
|
|
"metrics/advantage_var": 652.7297973632812,
|
|
"regularization/lipschitz_norm": 1676.2603759765625,
|
|
"regularization/mmd": 0.2351187914609909,
|
|
"regularization/rkhs_norm": 279.9033203125,
|
|
"regularization/w1": 1.408058762550354,
|
|
"rewards/chosen": 0.26928715907789597,
|
|
"rewards/margins": 0.7494250270484055,
|
|
"rewards/rejected": -0.48013786797050956,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 16.299654006958008,
|
|
"kl": 12.651652336120605,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -36233286.73155416,
|
|
"logits/rejected": -37188049.02021773,
|
|
"logps/chosen": -471.9628630298273,
|
|
"logps/rejected": -384.18895800933126,
|
|
"loss": 0.6139,
|
|
"loss/base_kto": 3.2932794094085693,
|
|
"metrics/advantage_var": 688.4408569335938,
|
|
"regularization/lipschitz_norm": 1650.3853759765625,
|
|
"regularization/mmd": 0.23184834420681,
|
|
"regularization/rkhs_norm": 276.00994873046875,
|
|
"regularization/w1": 1.3863238096237183,
|
|
"rewards/chosen": 0.39228669889680634,
|
|
"rewards/margins": 0.7795268580932002,
|
|
"rewards/rejected": -0.3872401591963939,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 44.51784133911133,
|
|
"kl": 13.391955375671387,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35239157.08852459,
|
|
"logits/rejected": -35575279.1880597,
|
|
"logps/chosen": -452.127612704918,
|
|
"logps/rejected": -357.58348880597015,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.3176803588867188,
|
|
"metrics/advantage_var": 589.2693481445312,
|
|
"regularization/lipschitz_norm": 1512.4068603515625,
|
|
"regularization/mmd": 0.21480341255664825,
|
|
"regularization/rkhs_norm": 255.7183380126953,
|
|
"regularization/w1": 1.2704216241836548,
|
|
"rewards/chosen": 0.40861958988377306,
|
|
"rewards/margins": 0.7385907667144633,
|
|
"rewards/rejected": -0.32997117683069027,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 20.914186477661133,
|
|
"kl": 11.465471267700195,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34505523.2,
|
|
"logits/rejected": -35888621.53442623,
|
|
"logps/chosen": -479.03083022388057,
|
|
"logps/rejected": -385.7593237704918,
|
|
"loss": 0.6158,
|
|
"loss/base_kto": 3.3036301136016846,
|
|
"metrics/advantage_var": 703.3262329101562,
|
|
"regularization/lipschitz_norm": 1652.409423828125,
|
|
"regularization/mmd": 0.23511762917041779,
|
|
"regularization/rkhs_norm": 279.90191650390625,
|
|
"regularization/w1": 1.3880237340927124,
|
|
"rewards/chosen": 0.40965526068388525,
|
|
"rewards/margins": 0.7906724015570281,
|
|
"rewards/rejected": -0.3810171408731429,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 13.190994262695312,
|
|
"kl": 15.52795696258545,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34810823.380726695,
|
|
"logits/rejected": -34491035.89489954,
|
|
"logps/chosen": -447.5599328593997,
|
|
"logps/rejected": -352.66219088098916,
|
|
"loss": 0.5864,
|
|
"loss/base_kto": 3.3103187084198,
|
|
"metrics/advantage_var": 468.60986328125,
|
|
"regularization/lipschitz_norm": 1403.2650146484375,
|
|
"regularization/mmd": 0.20243404805660248,
|
|
"regularization/rkhs_norm": 240.992919921875,
|
|
"regularization/w1": 1.1787426471710205,
|
|
"rewards/chosen": 0.4755921567220823,
|
|
"rewards/margins": 0.7135132096813535,
|
|
"rewards/rejected": -0.23792105295927116,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 18.810701370239258,
|
|
"kl": 8.278688430786133,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33646653.37539432,
|
|
"logits/rejected": -34206368.099071205,
|
|
"logps/chosen": -463.76744873817034,
|
|
"logps/rejected": -351.63332043343655,
|
|
"loss": 0.5884,
|
|
"loss/base_kto": 3.3078229427337646,
|
|
"metrics/advantage_var": 510.38311767578125,
|
|
"regularization/lipschitz_norm": 1417.2998046875,
|
|
"regularization/mmd": 0.2088085263967514,
|
|
"regularization/rkhs_norm": 248.58154296875,
|
|
"regularization/w1": 1.1905317306518555,
|
|
"rewards/chosen": 0.3218991252526124,
|
|
"rewards/margins": 0.7713713695625486,
|
|
"rewards/rejected": -0.44947224430993615,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 22.24349594116211,
|
|
"kl": 10.411376953125,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34501101.89067524,
|
|
"logits/rejected": -35971193.38601824,
|
|
"logps/chosen": -484.10515474276525,
|
|
"logps/rejected": -387.59002184650456,
|
|
"loss": 0.6128,
|
|
"loss/base_kto": 3.345282793045044,
|
|
"metrics/advantage_var": 638.2291870117188,
|
|
"regularization/lipschitz_norm": 1584.5711669921875,
|
|
"regularization/mmd": 0.22631387412548065,
|
|
"regularization/rkhs_norm": 269.4212341308594,
|
|
"regularization/w1": 1.3310397863388062,
|
|
"rewards/chosen": 0.28280440940734275,
|
|
"rewards/margins": 0.7016143871155969,
|
|
"rewards/rejected": -0.4188099777082542,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 21.045574188232422,
|
|
"kl": 7.60487699508667,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35705996.12631579,
|
|
"logits/rejected": -36351933.398373984,
|
|
"logps/chosen": -497.48336466165415,
|
|
"logps/rejected": -380.29024390243904,
|
|
"loss": 0.6088,
|
|
"loss/base_kto": 3.3214504718780518,
|
|
"metrics/advantage_var": 566.5302124023438,
|
|
"regularization/lipschitz_norm": 1581.8089599609375,
|
|
"regularization/mmd": 0.22012145817279816,
|
|
"regularization/rkhs_norm": 262.0493469238281,
|
|
"regularization/w1": 1.3287194967269897,
|
|
"rewards/chosen": 0.3303822223405193,
|
|
"rewards/margins": 0.7355862385530447,
|
|
"rewards/rejected": -0.4052040162125254,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.777828216552734,
|
|
"kl": 11.791571617126465,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -36184366.36834095,
|
|
"logits/rejected": -36301026.82504013,
|
|
"logps/chosen": -507.67489535768647,
|
|
"logps/rejected": -377.63678772070625,
|
|
"loss": 0.6161,
|
|
"loss/base_kto": 3.284566640853882,
|
|
"metrics/advantage_var": 640.5042114257812,
|
|
"regularization/lipschitz_norm": 1674.8494873046875,
|
|
"regularization/mmd": 0.2372153103351593,
|
|
"regularization/rkhs_norm": 282.399169921875,
|
|
"regularization/w1": 1.4068735837936401,
|
|
"rewards/chosen": 0.508470602050038,
|
|
"rewards/margins": 0.8046677003426995,
|
|
"rewards/rejected": -0.2961970982926615,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 19.283151626586914,
|
|
"kl": 18.055999755859375,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34655321.6,
|
|
"logits/rejected": -35655072.0,
|
|
"logps/chosen": -469.5294921875,
|
|
"logps/rejected": -372.74853515625,
|
|
"loss": 0.6006,
|
|
"loss/base_kto": 3.224091053009033,
|
|
"metrics/advantage_var": 673.1791381835938,
|
|
"regularization/lipschitz_norm": 1609.304931640625,
|
|
"regularization/mmd": 0.2292405217885971,
|
|
"regularization/rkhs_norm": 272.9053649902344,
|
|
"regularization/w1": 1.351816177368164,
|
|
"rewards/chosen": 0.5865845680236816,
|
|
"rewards/margins": 0.8517509937286377,
|
|
"rewards/rejected": -0.2651664257049561,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 22.157848358154297,
|
|
"kl": 11.102137565612793,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34884863.59300477,
|
|
"logits/rejected": -37201325.41935484,
|
|
"logps/chosen": -467.00720389507154,
|
|
"logps/rejected": -380.5667242703533,
|
|
"loss": 0.6033,
|
|
"loss/base_kto": 3.2924256324768066,
|
|
"metrics/advantage_var": 559.1273803710938,
|
|
"regularization/lipschitz_norm": 1563.8865966796875,
|
|
"regularization/mmd": 0.22006188333034515,
|
|
"regularization/rkhs_norm": 261.9784240722656,
|
|
"regularization/w1": 1.3136646747589111,
|
|
"rewards/chosen": 0.4102853553662808,
|
|
"rewards/margins": 0.7663093752785552,
|
|
"rewards/rejected": -0.3560240199122744,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 14.351923942565918,
|
|
"kl": 11.672685623168945,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36341127.668202765,
|
|
"logits/rejected": -36579481.03020668,
|
|
"logps/chosen": -475.1455453149002,
|
|
"logps/rejected": -389.1591563990461,
|
|
"loss": 0.5964,
|
|
"loss/base_kto": 3.276104688644409,
|
|
"metrics/advantage_var": 569.2327270507812,
|
|
"regularization/lipschitz_norm": 1519.04541015625,
|
|
"regularization/mmd": 0.21897660195827484,
|
|
"regularization/rkhs_norm": 260.6864318847656,
|
|
"regularization/w1": 1.2759981155395508,
|
|
"rewards/chosen": 0.3892057216661866,
|
|
"rewards/margins": 0.7845919490298094,
|
|
"rewards/rejected": -0.39538622736362283,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 19.687963485717773,
|
|
"kl": 12.144535064697266,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -36057032.2944,
|
|
"logits/rejected": -36286920.50076336,
|
|
"logps/chosen": -458.18145,
|
|
"logps/rejected": -379.506822519084,
|
|
"loss": 0.6089,
|
|
"loss/base_kto": 3.319847822189331,
|
|
"metrics/advantage_var": 591.9601440429688,
|
|
"regularization/lipschitz_norm": 1581.4017333984375,
|
|
"regularization/mmd": 0.22268171608448029,
|
|
"regularization/rkhs_norm": 265.0972900390625,
|
|
"regularization/w1": 1.328377604484558,
|
|
"rewards/chosen": 0.475669580078125,
|
|
"rewards/margins": 0.7342384268025406,
|
|
"rewards/rejected": -0.25856884672441555,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.162639617919922,
|
|
"kl": 11.866816520690918,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34904569.51898734,
|
|
"logits/rejected": -36339098.86419753,
|
|
"logps/chosen": -484.76839398734177,
|
|
"logps/rejected": -379.5061246141975,
|
|
"loss": 0.6085,
|
|
"loss/base_kto": 3.325211763381958,
|
|
"metrics/advantage_var": 568.21826171875,
|
|
"regularization/lipschitz_norm": 1576.9481201171875,
|
|
"regularization/mmd": 0.21834976971149445,
|
|
"regularization/rkhs_norm": 259.940185546875,
|
|
"regularization/w1": 1.3246363401412964,
|
|
"rewards/chosen": 0.3653219681751879,
|
|
"rewards/margins": 0.7204056221314716,
|
|
"rewards/rejected": -0.3550836539562838,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 20.01506805419922,
|
|
"kl": 10.111944198608398,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36305180.35569423,
|
|
"logits/rejected": -36879473.777777776,
|
|
"logps/chosen": -502.6751170046802,
|
|
"logps/rejected": -381.50376564945225,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.3790245056152344,
|
|
"metrics/advantage_var": 536.5336303710938,
|
|
"regularization/lipschitz_norm": 1439.1107177734375,
|
|
"regularization/mmd": 0.2128646820783615,
|
|
"regularization/rkhs_norm": 253.4103546142578,
|
|
"regularization/w1": 1.208853006362915,
|
|
"rewards/chosen": 0.3070826299848869,
|
|
"rewards/margins": 0.6483717860912298,
|
|
"rewards/rejected": -0.3412891561063429,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 20.250463485717773,
|
|
"kl": 15.722281455993652,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35885568.791344665,
|
|
"logits/rejected": -37519541.18167456,
|
|
"logps/chosen": -484.65880989180835,
|
|
"logps/rejected": -369.67797195892575,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.26762318611145,
|
|
"metrics/advantage_var": 602.1591186523438,
|
|
"regularization/lipschitz_norm": 1554.7772216796875,
|
|
"regularization/mmd": 0.2241850346326828,
|
|
"regularization/rkhs_norm": 266.886962890625,
|
|
"regularization/w1": 1.3060129880905151,
|
|
"rewards/chosen": 0.509227157191847,
|
|
"rewards/margins": 0.7842198484006246,
|
|
"rewards/rejected": -0.2749926912087777,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 23.537412643432617,
|
|
"kl": 11.714210510253906,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34093458.625387,
|
|
"logits/rejected": -35905865.48895899,
|
|
"logps/chosen": -478.8110487616099,
|
|
"logps/rejected": -390.30574723974763,
|
|
"loss": 0.604,
|
|
"loss/base_kto": 3.288358688354492,
|
|
"metrics/advantage_var": 583.552490234375,
|
|
"regularization/lipschitz_norm": 1569.8572998046875,
|
|
"regularization/mmd": 0.22495689988136292,
|
|
"regularization/rkhs_norm": 267.80584716796875,
|
|
"regularization/w1": 1.3186801671981812,
|
|
"rewards/chosen": 0.42627883769410313,
|
|
"rewards/margins": 0.7726709186810166,
|
|
"rewards/rejected": -0.34639208098691343,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 16.75432586669922,
|
|
"kl": 16.132680892944336,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34810250.79518072,
|
|
"logits/rejected": -35387857.45454545,
|
|
"logps/chosen": -460.7179028614458,
|
|
"logps/rejected": -382.6583806818182,
|
|
"loss": 0.6052,
|
|
"loss/base_kto": 3.27227783203125,
|
|
"metrics/advantage_var": 629.4932861328125,
|
|
"regularization/lipschitz_norm": 1595.4559326171875,
|
|
"regularization/mmd": 0.22874267399311066,
|
|
"regularization/rkhs_norm": 272.3127136230469,
|
|
"regularization/w1": 1.340182900428772,
|
|
"rewards/chosen": 0.558391800845962,
|
|
"rewards/margins": 0.7949300562154704,
|
|
"rewards/rejected": -0.23653825536950843,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 15.81540298461914,
|
|
"kl": 18.353281021118164,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34858116.592823714,
|
|
"logits/rejected": -36104593.88383046,
|
|
"logps/chosen": -486.6072542901716,
|
|
"logps/rejected": -363.5568337912088,
|
|
"loss": 0.5642,
|
|
"loss/base_kto": 3.2032318115234375,
|
|
"metrics/advantage_var": 473.7855529785156,
|
|
"regularization/lipschitz_norm": 1320.1123046875,
|
|
"regularization/mmd": 0.2014244794845581,
|
|
"regularization/rkhs_norm": 239.7910614013672,
|
|
"regularization/w1": 1.1088943481445312,
|
|
"rewards/chosen": 0.5943025538404348,
|
|
"rewards/margins": 0.8520283158887938,
|
|
"rewards/rejected": -0.257725762048359,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 23.31838035583496,
|
|
"kl": 17.869924545288086,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35392044.8,
|
|
"logits/rejected": -35940294.4,
|
|
"logps/chosen": -467.74482421875,
|
|
"logps/rejected": -394.97333984375,
|
|
"loss": 0.5584,
|
|
"loss/base_kto": 3.1466071605682373,
|
|
"metrics/advantage_var": 457.1605529785156,
|
|
"regularization/lipschitz_norm": 1330.1446533203125,
|
|
"regularization/mmd": 0.20303292572498322,
|
|
"regularization/rkhs_norm": 241.7058563232422,
|
|
"regularization/w1": 1.117321491241455,
|
|
"rewards/chosen": 0.6299750804901123,
|
|
"rewards/margins": 0.9154489040374756,
|
|
"rewards/rejected": -0.2854738235473633,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.353105545043945,
|
|
"kl": 16.537668228149414,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34650498.044728436,
|
|
"logits/rejected": -36549118.434250765,
|
|
"logps/chosen": -480.5283546325879,
|
|
"logps/rejected": -389.059250764526,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.1026206016540527,
|
|
"metrics/advantage_var": 481.5225524902344,
|
|
"regularization/lipschitz_norm": 1437.871826171875,
|
|
"regularization/mmd": 0.2070702612400055,
|
|
"regularization/rkhs_norm": 246.51220703125,
|
|
"regularization/w1": 1.2078123092651367,
|
|
"rewards/chosen": 0.6099318239254693,
|
|
"rewards/margins": 0.9424979714759997,
|
|
"rewards/rejected": -0.3325661475505304,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 15.787094116210938,
|
|
"kl": 11.99986457824707,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35630467.854199685,
|
|
"logits/rejected": -36641616.86286595,
|
|
"logps/chosen": -498.2772385103011,
|
|
"logps/rejected": -386.6381452234206,
|
|
"loss": 0.5518,
|
|
"loss/base_kto": 3.1225905418395996,
|
|
"metrics/advantage_var": 429.5301818847656,
|
|
"regularization/lipschitz_norm": 1302.1112060546875,
|
|
"regularization/mmd": 0.19768790900707245,
|
|
"regularization/rkhs_norm": 235.3427734375,
|
|
"regularization/w1": 1.0937732458114624,
|
|
"rewards/chosen": 0.5404235549660756,
|
|
"rewards/margins": 0.9109540228884803,
|
|
"rewards/rejected": -0.3705304679224047,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 17.42823028564453,
|
|
"kl": 12.829408645629883,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35645303.87915408,
|
|
"logits/rejected": -35891849.52750809,
|
|
"logps/chosen": -506.03148602719034,
|
|
"logps/rejected": -363.08902204692555,
|
|
"loss": 0.5657,
|
|
"loss/base_kto": 3.1499183177948,
|
|
"metrics/advantage_var": 435.5330505371094,
|
|
"regularization/lipschitz_norm": 1397.0950927734375,
|
|
"regularization/mmd": 0.20213496685028076,
|
|
"regularization/rkhs_norm": 240.63687133789062,
|
|
"regularization/w1": 1.1735600233078003,
|
|
"rewards/chosen": 0.5417508128186367,
|
|
"rewards/margins": 0.8809311264721813,
|
|
"rewards/rejected": -0.3391803136535447,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 17.52867317199707,
|
|
"kl": 12.546600341796875,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34529618.69551777,
|
|
"logits/rejected": -36097685.63665087,
|
|
"logps/chosen": -459.1051970633694,
|
|
"logps/rejected": -394.8283224723539,
|
|
"loss": 0.5578,
|
|
"loss/base_kto": 3.1983869075775146,
|
|
"metrics/advantage_var": 428.510986328125,
|
|
"regularization/lipschitz_norm": 1275.103515625,
|
|
"regularization/mmd": 0.19291190803050995,
|
|
"regularization/rkhs_norm": 229.6570281982422,
|
|
"regularization/w1": 1.0710868835449219,
|
|
"rewards/chosen": 0.4623830992803323,
|
|
"rewards/margins": 0.8338102767215843,
|
|
"rewards/rejected": -0.371427177441252,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 17.517013549804688,
|
|
"kl": 10.600077629089355,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35959843.73899848,
|
|
"logits/rejected": -37364665.09500805,
|
|
"logps/chosen": -481.2719081942337,
|
|
"logps/rejected": -378.451690821256,
|
|
"loss": 0.5554,
|
|
"loss/base_kto": 3.168569326400757,
|
|
"metrics/advantage_var": 399.3152770996094,
|
|
"regularization/lipschitz_norm": 1290.1065673828125,
|
|
"regularization/mmd": 0.1907900720834732,
|
|
"regularization/rkhs_norm": 227.1310272216797,
|
|
"regularization/w1": 1.083689570426941,
|
|
"rewards/chosen": 0.5179774005062121,
|
|
"rewards/margins": 0.8654425102161682,
|
|
"rewards/rejected": -0.3474651097099562,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 13.103388786315918,
|
|
"kl": 14.62479305267334,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34354505.37579618,
|
|
"logits/rejected": -36316967.26380368,
|
|
"logps/chosen": -488.8059315286624,
|
|
"logps/rejected": -359.2887030291411,
|
|
"loss": 0.5556,
|
|
"loss/base_kto": 3.127932071685791,
|
|
"metrics/advantage_var": 474.88775634765625,
|
|
"regularization/lipschitz_norm": 1326.002197265625,
|
|
"regularization/mmd": 0.20276470482349396,
|
|
"regularization/rkhs_norm": 241.3865509033203,
|
|
"regularization/w1": 1.1138417720794678,
|
|
"rewards/chosen": 0.6052237346673467,
|
|
"rewards/margins": 0.9245219230465525,
|
|
"rewards/rejected": -0.3192981883792058,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 15.394598007202148,
|
|
"kl": 15.183280944824219,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -35081658.469135806,
|
|
"logits/rejected": -35936631.898734175,
|
|
"logps/chosen": -516.3537808641976,
|
|
"logps/rejected": -385.87109375,
|
|
"loss": 0.566,
|
|
"loss/base_kto": 3.2061848640441895,
|
|
"metrics/advantage_var": 435.765625,
|
|
"regularization/lipschitz_norm": 1339.9744873046875,
|
|
"regularization/mmd": 0.1965966820716858,
|
|
"regularization/rkhs_norm": 234.04368591308594,
|
|
"regularization/w1": 1.12557852268219,
|
|
"rewards/chosen": 0.5497140766661844,
|
|
"rewards/margins": 0.8262719024548514,
|
|
"rewards/rejected": -0.27655782578866694,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 12.620918273925781,
|
|
"kl": 15.69377613067627,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34665299.48686244,
|
|
"logits/rejected": -35775589.91469194,
|
|
"logps/chosen": -472.3660162287481,
|
|
"logps/rejected": -372.7952705371248,
|
|
"loss": 0.5546,
|
|
"loss/base_kto": 3.1846084594726562,
|
|
"metrics/advantage_var": 419.4860534667969,
|
|
"regularization/lipschitz_norm": 1267.2525634765625,
|
|
"regularization/mmd": 0.18780146539211273,
|
|
"regularization/rkhs_norm": 223.5731658935547,
|
|
"regularization/w1": 1.064492106437683,
|
|
"rewards/chosen": 0.5874555490484931,
|
|
"rewards/margins": 0.8701356326502654,
|
|
"rewards/rejected": -0.2826800836017723,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 19.633522033691406,
|
|
"kl": 15.535196304321289,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35362592.05071315,
|
|
"logits/rejected": -37352572.64714946,
|
|
"logps/chosen": -475.3940174326466,
|
|
"logps/rejected": -369.34220916795067,
|
|
"loss": 0.5589,
|
|
"loss/base_kto": 3.159749984741211,
|
|
"metrics/advantage_var": 427.1222229003906,
|
|
"regularization/lipschitz_norm": 1331.1728515625,
|
|
"regularization/mmd": 0.19346921145915985,
|
|
"regularization/rkhs_norm": 230.3205108642578,
|
|
"regularization/w1": 1.11818528175354,
|
|
"rewards/chosen": 0.5518771744378962,
|
|
"rewards/margins": 0.8688022467772525,
|
|
"rewards/rejected": -0.31692507233935624,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 14.753633499145508,
|
|
"kl": 14.17564868927002,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34159447.2822186,
|
|
"logits/rejected": -36124054.83658171,
|
|
"logps/chosen": -472.7627956769984,
|
|
"logps/rejected": -363.40498500749624,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.173689603805542,
|
|
"metrics/advantage_var": 383.81085205078125,
|
|
"regularization/lipschitz_norm": 1270.6494140625,
|
|
"regularization/mmd": 0.18757760524749756,
|
|
"regularization/rkhs_norm": 223.3066864013672,
|
|
"regularization/w1": 1.0673454999923706,
|
|
"rewards/chosen": 0.502840175908697,
|
|
"rewards/margins": 0.8254180684809655,
|
|
"rewards/rejected": -0.32257789257226854,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 18.07467269897461,
|
|
"kl": 13.253885269165039,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34877716.06269593,
|
|
"logits/rejected": -35838491.115264796,
|
|
"logps/chosen": -481.9712970219436,
|
|
"logps/rejected": -382.05561234423675,
|
|
"loss": 0.5533,
|
|
"loss/base_kto": 3.142148971557617,
|
|
"metrics/advantage_var": 422.1423034667969,
|
|
"regularization/lipschitz_norm": 1298.2183837890625,
|
|
"regularization/mmd": 0.19337554275989532,
|
|
"regularization/rkhs_norm": 230.20896911621094,
|
|
"regularization/w1": 1.0905033349990845,
|
|
"rewards/chosen": 0.5188090644286344,
|
|
"rewards/margins": 0.8925310814256895,
|
|
"rewards/rejected": -0.3737220169970551,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 21.723722457885742,
|
|
"kl": 12.508740425109863,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36401694.76733436,
|
|
"logits/rejected": -36952784.532488115,
|
|
"logps/chosen": -493.67421032357475,
|
|
"logps/rejected": -395.0083201267829,
|
|
"loss": 0.5598,
|
|
"loss/base_kto": 3.1529204845428467,
|
|
"metrics/advantage_var": 457.195068359375,
|
|
"regularization/lipschitz_norm": 1336.2327880859375,
|
|
"regularization/mmd": 0.20305581390857697,
|
|
"regularization/rkhs_norm": 241.7331085205078,
|
|
"regularization/w1": 1.122435450553894,
|
|
"rewards/chosen": 0.5538677103897823,
|
|
"rewards/margins": 0.880019538581331,
|
|
"rewards/rejected": -0.32615182819154864,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 18.0310001373291,
|
|
"kl": 17.11049461364746,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33666531.86813187,
|
|
"logits/rejected": -35667910.66874028,
|
|
"logps/chosen": -486.9624215070644,
|
|
"logps/rejected": -389.0480171073095,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.180561065673828,
|
|
"metrics/advantage_var": 428.9582214355469,
|
|
"regularization/lipschitz_norm": 1298.237060546875,
|
|
"regularization/mmd": 0.19746464490890503,
|
|
"regularization/rkhs_norm": 235.0769500732422,
|
|
"regularization/w1": 1.0905191898345947,
|
|
"rewards/chosen": 0.5803600173542975,
|
|
"rewards/margins": 0.8641187236443928,
|
|
"rewards/rejected": -0.28375870629009525,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 14.83030891418457,
|
|
"kl": 15.62182903289795,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33895261.61812298,
|
|
"logits/rejected": -34788599.49244713,
|
|
"logps/chosen": -461.1163531553398,
|
|
"logps/rejected": -375.5492116691843,
|
|
"loss": 0.5571,
|
|
"loss/base_kto": 3.197308301925659,
|
|
"metrics/advantage_var": 401.8186950683594,
|
|
"regularization/lipschitz_norm": 1272.59765625,
|
|
"regularization/mmd": 0.19013570249080658,
|
|
"regularization/rkhs_norm": 226.35203552246094,
|
|
"regularization/w1": 1.0689821243286133,
|
|
"rewards/chosen": 0.5586881668436489,
|
|
"rewards/margins": 0.842465340395917,
|
|
"rewards/rejected": -0.2837771735522682,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 17.9812068939209,
|
|
"kl": 11.873516082763672,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34445551.53216374,
|
|
"logits/rejected": -36125878.12080537,
|
|
"logps/chosen": -456.01667580409355,
|
|
"logps/rejected": -363.6667627936242,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.160029172897339,
|
|
"metrics/advantage_var": 426.356689453125,
|
|
"regularization/lipschitz_norm": 1331.6898193359375,
|
|
"regularization/mmd": 0.19539673626422882,
|
|
"regularization/rkhs_norm": 232.6151580810547,
|
|
"regularization/w1": 1.1186193227767944,
|
|
"rewards/chosen": 0.5473325852065059,
|
|
"rewards/margins": 0.8867866216375304,
|
|
"rewards/rejected": -0.33945403643102456,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 15.903279304504395,
|
|
"kl": 15.79315185546875,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35280485.73828756,
|
|
"logits/rejected": -35382403.67927383,
|
|
"logps/chosen": -476.8421344911147,
|
|
"logps/rejected": -373.09093702723146,
|
|
"loss": 0.5538,
|
|
"loss/base_kto": 3.1853463649749756,
|
|
"metrics/advantage_var": 387.84149169921875,
|
|
"regularization/lipschitz_norm": 1260.2186279296875,
|
|
"regularization/mmd": 0.18613950908184052,
|
|
"regularization/rkhs_norm": 221.5946502685547,
|
|
"regularization/w1": 1.0585834980010986,
|
|
"rewards/chosen": 0.49597148248182554,
|
|
"rewards/margins": 0.8129807432048068,
|
|
"rewards/rejected": -0.3170092607229813,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 18.10071563720703,
|
|
"kl": 13.327929496765137,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35143539.7260274,
|
|
"logits/rejected": -36358923.09470305,
|
|
"logps/chosen": -481.94411149162863,
|
|
"logps/rejected": -378.6756370385233,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.1585705280303955,
|
|
"metrics/advantage_var": 449.4476623535156,
|
|
"regularization/lipschitz_norm": 1374.4207763671875,
|
|
"regularization/mmd": 0.20149266719818115,
|
|
"regularization/rkhs_norm": 239.8722381591797,
|
|
"regularization/w1": 1.1545134782791138,
|
|
"rewards/chosen": 0.5620129264471556,
|
|
"rewards/margins": 0.8867209373946877,
|
|
"rewards/rejected": -0.3247080109475321,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 18.524930953979492,
|
|
"kl": 15.759302139282227,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -36137202.12807882,
|
|
"logits/rejected": -36938408.631892696,
|
|
"logps/chosen": -478.5948788998358,
|
|
"logps/rejected": -379.47571255588673,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.1908624172210693,
|
|
"metrics/advantage_var": 423.966064453125,
|
|
"regularization/lipschitz_norm": 1271.6673583984375,
|
|
"regularization/mmd": 0.1926002949476242,
|
|
"regularization/rkhs_norm": 229.2860870361328,
|
|
"regularization/w1": 1.0682004690170288,
|
|
"rewards/chosen": 0.5441734176159688,
|
|
"rewards/margins": 0.8334453564345465,
|
|
"rewards/rejected": -0.28927193881857766,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 17.577173233032227,
|
|
"kl": 10.820630073547363,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35068814.222222224,
|
|
"logits/rejected": -36099434.93670886,
|
|
"logps/chosen": -503.60108024691357,
|
|
"logps/rejected": -366.8716376582278,
|
|
"loss": 0.5561,
|
|
"loss/base_kto": 3.1237423419952393,
|
|
"metrics/advantage_var": 437.7642517089844,
|
|
"regularization/lipschitz_norm": 1340.9012451171875,
|
|
"regularization/mmd": 0.1989518254995346,
|
|
"regularization/rkhs_norm": 236.847412109375,
|
|
"regularization/w1": 1.126357078552246,
|
|
"rewards/chosen": 0.5330301449622636,
|
|
"rewards/margins": 0.9208744720921589,
|
|
"rewards/rejected": -0.3878443271298952,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 12.616131782531738,
|
|
"kl": 14.004104614257812,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34379226.691472866,
|
|
"logits/rejected": -37020938.07874016,
|
|
"logps/chosen": -482.3362403100775,
|
|
"logps/rejected": -365.29397145669293,
|
|
"loss": 0.5512,
|
|
"loss/base_kto": 3.1323635578155518,
|
|
"metrics/advantage_var": 427.05615234375,
|
|
"regularization/lipschitz_norm": 1289.6724853515625,
|
|
"regularization/mmd": 0.19414694607257843,
|
|
"regularization/rkhs_norm": 231.1273193359375,
|
|
"regularization/w1": 1.083324909210205,
|
|
"rewards/chosen": 0.5665439339571221,
|
|
"rewards/margins": 0.9168358454052028,
|
|
"rewards/rejected": -0.3502919114480807,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 13.249449729919434,
|
|
"kl": 14.09144115447998,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34104844.641975306,
|
|
"logits/rejected": -35981149.974683546,
|
|
"logps/chosen": -474.2334587191358,
|
|
"logps/rejected": -359.50118670886076,
|
|
"loss": 0.5558,
|
|
"loss/base_kto": 3.192821979522705,
|
|
"metrics/advantage_var": 428.54833984375,
|
|
"regularization/lipschitz_norm": 1260.3631591796875,
|
|
"regularization/mmd": 0.19503657519817352,
|
|
"regularization/rkhs_norm": 232.18638610839844,
|
|
"regularization/w1": 1.0587049722671509,
|
|
"rewards/chosen": 0.5502282601815683,
|
|
"rewards/margins": 0.8538953286630137,
|
|
"rewards/rejected": -0.3036670684814453,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 17.073501586914062,
|
|
"kl": 12.076838493347168,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35473696.820512824,
|
|
"logits/rejected": -36427654.24390244,
|
|
"logps/chosen": -478.97075320512823,
|
|
"logps/rejected": -382.3941739710366,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.156153440475464,
|
|
"metrics/advantage_var": 425.79400634765625,
|
|
"regularization/lipschitz_norm": 1337.58935546875,
|
|
"regularization/mmd": 0.19765332341194153,
|
|
"regularization/rkhs_norm": 235.3015594482422,
|
|
"regularization/w1": 1.1235750913619995,
|
|
"rewards/chosen": 0.5304658351800381,
|
|
"rewards/margins": 0.9001460537603305,
|
|
"rewards/rejected": -0.3696802185802925,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 15.593650817871094,
|
|
"kl": 10.039626121520996,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35000255.597484276,
|
|
"logits/rejected": -36944151.850931674,
|
|
"logps/chosen": -499.43828616352204,
|
|
"logps/rejected": -360.04386645962734,
|
|
"loss": 0.5531,
|
|
"loss/base_kto": 3.1356537342071533,
|
|
"metrics/advantage_var": 429.6227722167969,
|
|
"regularization/lipschitz_norm": 1302.9326171875,
|
|
"regularization/mmd": 0.1944286823272705,
|
|
"regularization/rkhs_norm": 231.46275329589844,
|
|
"regularization/w1": 1.0944633483886719,
|
|
"rewards/chosen": 0.5186354439213591,
|
|
"rewards/margins": 0.9125994176360572,
|
|
"rewards/rejected": -0.39396397371469816,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 14.626014709472656,
|
|
"kl": 10.720664978027344,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34507064.67175572,
|
|
"logits/rejected": -34558089.6256,
|
|
"logps/chosen": -492.1023854961832,
|
|
"logps/rejected": -365.07745,
|
|
"loss": 0.5549,
|
|
"loss/base_kto": 3.1620523929595947,
|
|
"metrics/advantage_var": 409.95391845703125,
|
|
"regularization/lipschitz_norm": 1290.7791748046875,
|
|
"regularization/mmd": 0.19280944764614105,
|
|
"regularization/rkhs_norm": 229.53506469726562,
|
|
"regularization/w1": 1.0842543840408325,
|
|
"rewards/chosen": 0.5193258736879771,
|
|
"rewards/margins": 0.8730520699770397,
|
|
"rewards/rejected": -0.3537261962890625,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 17.921890258789062,
|
|
"kl": 10.978423118591309,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34726710.326717556,
|
|
"logits/rejected": -35583764.0704,
|
|
"logps/chosen": -481.82471374045804,
|
|
"logps/rejected": -361.47715,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.2089245319366455,
|
|
"metrics/advantage_var": 391.92059326171875,
|
|
"regularization/lipschitz_norm": 1256.5316162109375,
|
|
"regularization/mmd": 0.18653170764446259,
|
|
"regularization/rkhs_norm": 222.0615692138672,
|
|
"regularization/w1": 1.0554865598678589,
|
|
"rewards/chosen": 0.4799011696385973,
|
|
"rewards/margins": 0.8107483864354723,
|
|
"rewards/rejected": -0.330847216796875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 13.252175331115723,
|
|
"kl": 11.777840614318848,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35449815.53799392,
|
|
"logits/rejected": -36389180.09003215,
|
|
"logps/chosen": -448.4937310030395,
|
|
"logps/rejected": -357.3993167202572,
|
|
"loss": 0.559,
|
|
"loss/base_kto": 3.210728406906128,
|
|
"metrics/advantage_var": 405.4601745605469,
|
|
"regularization/lipschitz_norm": 1274.8612060546875,
|
|
"regularization/mmd": 0.19031064212322235,
|
|
"regularization/rkhs_norm": 226.5602569580078,
|
|
"regularization/w1": 1.0708832740783691,
|
|
"rewards/chosen": 0.5015448016598594,
|
|
"rewards/margins": 0.8342519277007934,
|
|
"rewards/rejected": -0.332707126040934,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 20.83140754699707,
|
|
"kl": 12.857622146606445,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -35669300.58170915,
|
|
"logits/rejected": -38086579.15823817,
|
|
"logps/chosen": -478.5110569715142,
|
|
"logps/rejected": -384.370845228385,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.1982758045196533,
|
|
"metrics/advantage_var": 423.3090515136719,
|
|
"regularization/lipschitz_norm": 1306.4466552734375,
|
|
"regularization/mmd": 0.19334715604782104,
|
|
"regularization/rkhs_norm": 230.17520141601562,
|
|
"regularization/w1": 1.0974153280258179,
|
|
"rewards/chosen": 0.522906896771341,
|
|
"rewards/margins": 0.8397517436119303,
|
|
"rewards/rejected": -0.31684484684058933,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 17.806838989257812,
|
|
"kl": 12.624212265014648,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35652015.15789474,
|
|
"logits/rejected": -35866480.80650406,
|
|
"logps/chosen": -449.30112781954887,
|
|
"logps/rejected": -393.7626524390244,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.2325809001922607,
|
|
"metrics/advantage_var": 437.7423400878906,
|
|
"regularization/lipschitz_norm": 1292.982666015625,
|
|
"regularization/mmd": 0.19358959794044495,
|
|
"regularization/rkhs_norm": 230.4638214111328,
|
|
"regularization/w1": 1.086105465888977,
|
|
"rewards/chosen": 0.5171561563821664,
|
|
"rewards/margins": 0.8115294096850119,
|
|
"rewards/rejected": -0.29437325330284553,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 16.46271324157715,
|
|
"kl": 15.088406562805176,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34679534.289308175,
|
|
"logits/rejected": -35387996.22360248,
|
|
"logps/chosen": -488.41568396226415,
|
|
"logps/rejected": -389.755119371118,
|
|
"loss": 0.5622,
|
|
"loss/base_kto": 3.1939010620117188,
|
|
"metrics/advantage_var": 425.0210876464844,
|
|
"regularization/lipschitz_norm": 1319.3656005859375,
|
|
"regularization/mmd": 0.1957203596830368,
|
|
"regularization/rkhs_norm": 233.0004425048828,
|
|
"regularization/w1": 1.108267068862915,
|
|
"rewards/chosen": 0.5672503297433913,
|
|
"rewards/margins": 0.8323033147037222,
|
|
"rewards/rejected": -0.2650529849603309,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 18.391647338867188,
|
|
"kl": 16.242544174194336,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -33360569.812995244,
|
|
"logits/rejected": -34305943.86440678,
|
|
"logps/chosen": -485.9695919175911,
|
|
"logps/rejected": -370.52768682588595,
|
|
"loss": 0.5551,
|
|
"loss/base_kto": 3.15459942817688,
|
|
"metrics/advantage_var": 418.97021484375,
|
|
"regularization/lipschitz_norm": 1302.6507568359375,
|
|
"regularization/mmd": 0.1921084225177765,
|
|
"regularization/rkhs_norm": 228.7004852294922,
|
|
"regularization/w1": 1.094226598739624,
|
|
"rewards/chosen": 0.5278950976872029,
|
|
"rewards/margins": 0.8850346028002374,
|
|
"rewards/rejected": -0.3571395051130345,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 18.19830322265625,
|
|
"kl": 15.978507995605469,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34110690.119873814,
|
|
"logits/rejected": -36318566.241486065,
|
|
"logps/chosen": -487.15048304416405,
|
|
"logps/rejected": -373.83995259287923,
|
|
"loss": 0.551,
|
|
"loss/base_kto": 3.2148778438568115,
|
|
"metrics/advantage_var": 366.65606689453125,
|
|
"regularization/lipschitz_norm": 1204.7899169921875,
|
|
"regularization/mmd": 0.1810026317834854,
|
|
"regularization/rkhs_norm": 215.4793243408203,
|
|
"regularization/w1": 1.0120233297348022,
|
|
"rewards/chosen": 0.49806439613318215,
|
|
"rewards/margins": 0.7901091626803232,
|
|
"rewards/rejected": -0.29204476654714107,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 15.575223922729492,
|
|
"kl": 13.734251976013184,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34421687.53230769,
|
|
"logits/rejected": -36010344.83809524,
|
|
"logps/chosen": -473.1840384615385,
|
|
"logps/rejected": -378.83397817460315,
|
|
"loss": 0.5558,
|
|
"loss/base_kto": 3.1955902576446533,
|
|
"metrics/advantage_var": 399.99322509765625,
|
|
"regularization/lipschitz_norm": 1263.6273193359375,
|
|
"regularization/mmd": 0.18925951421260834,
|
|
"regularization/rkhs_norm": 225.30894470214844,
|
|
"regularization/w1": 1.061447024345398,
|
|
"rewards/chosen": 0.5103482759915865,
|
|
"rewards/margins": 0.8328501457987542,
|
|
"rewards/rejected": -0.32250186980716766,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 22.56538963317871,
|
|
"kl": 16.62363624572754,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -33329488.338211384,
|
|
"logits/rejected": -35892006.88120301,
|
|
"logps/chosen": -486.17525406504063,
|
|
"logps/rejected": -366.10726033834584,
|
|
"loss": 0.5533,
|
|
"loss/base_kto": 3.182363986968994,
|
|
"metrics/advantage_var": 396.3825988769531,
|
|
"regularization/lipschitz_norm": 1256.23046875,
|
|
"regularization/mmd": 0.18909044563770294,
|
|
"regularization/rkhs_norm": 225.107666015625,
|
|
"regularization/w1": 1.0552337169647217,
|
|
"rewards/chosen": 0.543873062754065,
|
|
"rewards/margins": 0.8428124048226153,
|
|
"rewards/rejected": -0.2989393420685503,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 18.93718719482422,
|
|
"kl": 14.416910171508789,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35279206.15580286,
|
|
"logits/rejected": -37080021.52995392,
|
|
"logps/chosen": -464.6409479332273,
|
|
"logps/rejected": -378.84190188172045,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.1653780937194824,
|
|
"metrics/advantage_var": 439.2682800292969,
|
|
"regularization/lipschitz_norm": 1352.9263916015625,
|
|
"regularization/mmd": 0.19671465456485748,
|
|
"regularization/rkhs_norm": 234.1841278076172,
|
|
"regularization/w1": 1.1364582777023315,
|
|
"rewards/chosen": 0.5402929551650686,
|
|
"rewards/margins": 0.8828776852752956,
|
|
"rewards/rejected": -0.34258473011022705,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 19.302715301513672,
|
|
"kl": 14.864020347595215,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -36242167.48019017,
|
|
"logits/rejected": -36907796.90600924,
|
|
"logps/chosen": -508.99569136291603,
|
|
"logps/rejected": -352.2167276579353,
|
|
"loss": 0.5634,
|
|
"loss/base_kto": 3.1400930881500244,
|
|
"metrics/advantage_var": 449.7011413574219,
|
|
"regularization/lipschitz_norm": 1387.0028076171875,
|
|
"regularization/mmd": 0.20231874287128448,
|
|
"regularization/rkhs_norm": 240.8556671142578,
|
|
"regularization/w1": 1.1650822162628174,
|
|
"rewards/chosen": 0.5618764934751387,
|
|
"rewards/margins": 0.9130513873172275,
|
|
"rewards/rejected": -0.3511748938420888,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 16.154787063598633,
|
|
"kl": 15.28672981262207,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -36164415.494470775,
|
|
"logits/rejected": -36540536.28438949,
|
|
"logps/chosen": -488.46978672985784,
|
|
"logps/rejected": -391.0024874420402,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.1421260833740234,
|
|
"metrics/advantage_var": 457.24761962890625,
|
|
"regularization/lipschitz_norm": 1342.6614990234375,
|
|
"regularization/mmd": 0.19898290932178497,
|
|
"regularization/rkhs_norm": 236.8843994140625,
|
|
"regularization/w1": 1.1278356313705444,
|
|
"rewards/chosen": 0.5228059355869866,
|
|
"rewards/margins": 0.8927053492721441,
|
|
"rewards/rejected": -0.36989941368515744,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_loss": 0.5863481292658725,
|
|
"train_runtime": 11098.6525,
|
|
"train_samples_per_second": 13.355,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|