2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 18.29014778137207,
|
||
|
|
"kl": 1.4570707082748413,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37126905.17333333,
|
||
|
|
"logits/rejected": -39094076.23529412,
|
||
|
|
"logps/chosen": -485.8353125,
|
||
|
|
"logps/rejected": -357.3402573529412,
|
||
|
|
"loss": 0.5972,
|
||
|
|
"loss/base_kto": 3.8672566413879395,
|
||
|
|
"metrics/advantage_var": 195.62730407714844,
|
||
|
|
"regularization/lipschitz_norm": 940.6920776367188,
|
||
|
|
"regularization/w1": 0.910589873790741,
|
||
|
|
"rewards/chosen": -0.2204015096028646,
|
||
|
|
"rewards/margins": 0.10056833005418964,
|
||
|
|
"rewards/rejected": -0.3209698396570542,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 22.535978317260742,
|
||
|
|
"kl": 2.5399329662323,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -35308072.99215071,
|
||
|
|
"logits/rejected": -36455594.40124417,
|
||
|
|
"logps/chosen": -465.87078100470956,
|
||
|
|
"logps/rejected": -380.4113287325039,
|
||
|
|
"loss": 0.5878,
|
||
|
|
"loss/base_kto": 3.904927968978882,
|
||
|
|
"metrics/advantage_var": 149.05447387695312,
|
||
|
|
"regularization/lipschitz_norm": 823.9749145507812,
|
||
|
|
"regularization/w1": 0.7976077198982239,
|
||
|
|
"rewards/chosen": -0.06399994255796519,
|
||
|
|
"rewards/margins": 0.08554940821022348,
|
||
|
|
"rewards/rejected": -0.14954935076818868,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 24.34197235107422,
|
||
|
|
"kl": 3.1339657306671143,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -34763361.8343949,
|
||
|
|
"logits/rejected": -37482037.39877301,
|
||
|
|
"logps/chosen": -482.2357683121019,
|
||
|
|
"logps/rejected": -378.5544238880368,
|
||
|
|
"loss": 0.5942,
|
||
|
|
"loss/base_kto": 3.8586366176605225,
|
||
|
|
"metrics/advantage_var": 188.65928649902344,
|
||
|
|
"regularization/lipschitz_norm": 924.5841064453125,
|
||
|
|
"regularization/w1": 0.8949974179267883,
|
||
|
|
"rewards/chosen": -0.00727593291337323,
|
||
|
|
"rewards/margins": 0.1317377202391862,
|
||
|
|
"rewards/rejected": -0.13901365315255942,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 27.18002700805664,
|
||
|
|
"kl": 2.128688335418701,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -36121831.0144,
|
||
|
|
"logits/rejected": -37452341.93587786,
|
||
|
|
"logps/chosen": -480.60845,
|
||
|
|
"logps/rejected": -372.37197041984734,
|
||
|
|
"loss": 0.5966,
|
||
|
|
"loss/base_kto": 3.922574758529663,
|
||
|
|
"metrics/advantage_var": 185.8826904296875,
|
||
|
|
"regularization/lipschitz_norm": 878.3036499023438,
|
||
|
|
"regularization/w1": 0.8501979112625122,
|
||
|
|
"rewards/chosen": -0.0865505859375,
|
||
|
|
"rewards/margins": 0.0659774343243082,
|
||
|
|
"rewards/rejected": -0.1525280202618082,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 23.27158546447754,
|
||
|
|
"kl": 4.320512294769287,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -35060289.55591572,
|
||
|
|
"logits/rejected": -36571040.2413273,
|
||
|
|
"logps/chosen": -448.2167747163695,
|
||
|
|
"logps/rejected": -361.9203431372549,
|
||
|
|
"loss": 0.5886,
|
||
|
|
"loss/base_kto": 3.891024112701416,
|
||
|
|
"metrics/advantage_var": 155.9692840576172,
|
||
|
|
"regularization/lipschitz_norm": 845.15283203125,
|
||
|
|
"regularization/w1": 0.8181080222129822,
|
||
|
|
"rewards/chosen": -0.028017547953650475,
|
||
|
|
"rewards/margins": 0.09802821994410357,
|
||
|
|
"rewards/rejected": -0.12604576789775404,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 23.44509506225586,
|
||
|
|
"kl": 9.313093185424805,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -35588562.823529415,
|
||
|
|
"logits/rejected": -36494690.986666664,
|
||
|
|
"logps/chosen": -482.75128676470587,
|
||
|
|
"logps/rejected": -368.330546875,
|
||
|
|
"loss": 0.5954,
|
||
|
|
"loss/base_kto": 3.8357720375061035,
|
||
|
|
"metrics/advantage_var": 211.01597595214844,
|
||
|
|
"regularization/lipschitz_norm": 958.1713256835938,
|
||
|
|
"regularization/w1": 0.9275097250938416,
|
||
|
|
"rewards/chosen": 0.1270505792954389,
|
||
|
|
"rewards/margins": 0.16840975593118107,
|
||
|
|
"rewards/rejected": -0.041359176635742186,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 27.102447509765625,
|
||
|
|
"kl": 26.627674102783203,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -35467575.78507463,
|
||
|
|
"logits/rejected": -36844597.718032785,
|
||
|
|
"logps/chosen": -472.9744402985075,
|
||
|
|
"logps/rejected": -354.23683401639346,
|
||
|
|
"loss": 0.5967,
|
||
|
|
"loss/base_kto": 3.7943520545959473,
|
||
|
|
"metrics/advantage_var": 217.12857055664062,
|
||
|
|
"regularization/lipschitz_norm": 1011.2306518554688,
|
||
|
|
"regularization/w1": 0.9788713455200195,
|
||
|
|
"rewards/chosen": 0.45366547997318096,
|
||
|
|
"rewards/margins": 0.18523941656065535,
|
||
|
|
"rewards/rejected": 0.2684260634125256,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 22.07541847229004,
|
||
|
|
"kl": 29.8513240814209,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -37256623.49685535,
|
||
|
|
"logits/rejected": -36983947.92546584,
|
||
|
|
"logps/chosen": -471.38109276729557,
|
||
|
|
"logps/rejected": -361.8736413043478,
|
||
|
|
"loss": 0.5901,
|
||
|
|
"loss/base_kto": 3.7140281200408936,
|
||
|
|
"metrics/advantage_var": 231.1140594482422,
|
||
|
|
"regularization/lipschitz_norm": 1039.8240966796875,
|
||
|
|
"regularization/w1": 1.006549596786499,
|
||
|
|
"rewards/chosen": 0.4847293110013758,
|
||
|
|
"rewards/margins": 0.25128047684868027,
|
||
|
|
"rewards/rejected": 0.23344883415269554,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 21.419479370117188,
|
||
|
|
"kl": 14.638226509094238,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -36142804.99843014,
|
||
|
|
"logits/rejected": -37287455.05443235,
|
||
|
|
"logps/chosen": -471.28051412872844,
|
||
|
|
"logps/rejected": -347.9593944401244,
|
||
|
|
"loss": 0.5901,
|
||
|
|
"loss/base_kto": 3.77160906791687,
|
||
|
|
"metrics/advantage_var": 220.82177734375,
|
||
|
|
"regularization/lipschitz_norm": 980.3154296875,
|
||
|
|
"regularization/w1": 0.9489454627037048,
|
||
|
|
"rewards/chosen": 0.20925064925309067,
|
||
|
|
"rewards/margins": 0.19540523370761673,
|
||
|
|
"rewards/rejected": 0.013845415545473944,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 22.473234176635742,
|
||
|
|
"kl": 25.710744857788086,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -38515946.86238532,
|
||
|
|
"logits/rejected": -38492212.34504792,
|
||
|
|
"logps/chosen": -507.7592698776758,
|
||
|
|
"logps/rejected": -360.59669528753994,
|
||
|
|
"loss": 0.595,
|
||
|
|
"loss/base_kto": 3.7333428859710693,
|
||
|
|
"metrics/advantage_var": 292.5412292480469,
|
||
|
|
"regularization/lipschitz_norm": 1060.6162109375,
|
||
|
|
"regularization/w1": 1.0266764163970947,
|
||
|
|
"rewards/chosen": 0.44321611970207375,
|
||
|
|
"rewards/margins": 0.22823484560001206,
|
||
|
|
"rewards/rejected": 0.2149812741020617,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 21.400148391723633,
|
||
|
|
"kl": 29.867273330688477,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37405573.24430956,
|
||
|
|
"logits/rejected": -39347989.84863124,
|
||
|
|
"logps/chosen": -495.8902219271624,
|
||
|
|
"logps/rejected": -373.7242351046699,
|
||
|
|
"loss": 0.5995,
|
||
|
|
"loss/base_kto": 3.8104615211486816,
|
||
|
|
"metrics/advantage_var": 244.73912048339844,
|
||
|
|
"regularization/lipschitz_norm": 1018.1398315429688,
|
||
|
|
"regularization/w1": 0.9855592846870422,
|
||
|
|
"rewards/chosen": 0.4900019042227807,
|
||
|
|
"rewards/margins": 0.16669079417315058,
|
||
|
|
"rewards/rejected": 0.32331111004963015,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 22.627262115478516,
|
||
|
|
"kl": 18.140928268432617,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -36578178.189735614,
|
||
|
|
"logits/rejected": -37561258.80062795,
|
||
|
|
"logps/chosen": -474.09603421461895,
|
||
|
|
"logps/rejected": -365.09205749607537,
|
||
|
|
"loss": 0.5939,
|
||
|
|
"loss/base_kto": 3.809729814529419,
|
||
|
|
"metrics/advantage_var": 220.01670837402344,
|
||
|
|
"regularization/lipschitz_norm": 972.9920043945312,
|
||
|
|
"regularization/w1": 0.9418562054634094,
|
||
|
|
"rewards/chosen": 0.2615461809268079,
|
||
|
|
"rewards/margins": 0.15695515863297455,
|
||
|
|
"rewards/rejected": 0.1045910222938334,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 20.957517623901367,
|
||
|
|
"kl": 6.641819953918457,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -37125660.22047244,
|
||
|
|
"logits/rejected": -38276694.524031006,
|
||
|
|
"logps/chosen": -488.0752952755906,
|
||
|
|
"logps/rejected": -380.35503875968993,
|
||
|
|
"loss": 0.6034,
|
||
|
|
"loss/base_kto": 3.8500633239746094,
|
||
|
|
"metrics/advantage_var": 227.3952178955078,
|
||
|
|
"regularization/lipschitz_norm": 1009.8142700195312,
|
||
|
|
"regularization/w1": 0.9775001406669617,
|
||
|
|
"rewards/chosen": 0.048197618619663506,
|
||
|
|
"rewards/margins": 0.1383724311139858,
|
||
|
|
"rewards/rejected": -0.09017481249432231,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 18.912769317626953,
|
||
|
|
"kl": 9.864899635314941,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -35173403.8528,
|
||
|
|
"logits/rejected": -37522114.63816794,
|
||
|
|
"logps/chosen": -479.1098,
|
||
|
|
"logps/rejected": -384.2582776717557,
|
||
|
|
"loss": 0.5971,
|
||
|
|
"loss/base_kto": 3.7604424953460693,
|
||
|
|
"metrics/advantage_var": 254.92312622070312,
|
||
|
|
"regularization/lipschitz_norm": 1049.9703369140625,
|
||
|
|
"regularization/w1": 1.0163713693618774,
|
||
|
|
"rewards/chosen": 0.09310198974609375,
|
||
|
|
"rewards/margins": 0.18677775166052898,
|
||
|
|
"rewards/rejected": -0.09367576191443523,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 22.78107261657715,
|
||
|
|
"kl": 7.072554111480713,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -36721923.16049383,
|
||
|
|
"logits/rejected": -37791792.60759494,
|
||
|
|
"logps/chosen": -482.63917824074076,
|
||
|
|
"logps/rejected": -391.1912579113924,
|
||
|
|
"loss": 0.6038,
|
||
|
|
"loss/base_kto": 3.808419942855835,
|
||
|
|
"metrics/advantage_var": 250.69656372070312,
|
||
|
|
"regularization/lipschitz_norm": 1055.9146728515625,
|
||
|
|
"regularization/w1": 1.022125244140625,
|
||
|
|
"rewards/chosen": 0.09606993639910663,
|
||
|
|
"rewards/margins": 0.1902502245932077,
|
||
|
|
"rewards/rejected": -0.09418028819410107,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 23.265239715576172,
|
||
|
|
"kl": 15.797464370727539,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -38567671.14586466,
|
||
|
|
"logits/rejected": -38698278.71219512,
|
||
|
|
"logps/chosen": -468.85075187969926,
|
||
|
|
"logps/rejected": -387.3363821138211,
|
||
|
|
"loss": 0.5941,
|
||
|
|
"loss/base_kto": 3.741375684738159,
|
||
|
|
"metrics/advantage_var": 256.41436767578125,
|
||
|
|
"regularization/lipschitz_norm": 1044.8873291015625,
|
||
|
|
"regularization/w1": 1.0114508867263794,
|
||
|
|
"rewards/chosen": 0.3573325250381814,
|
||
|
|
"rewards/margins": 0.24188839751341007,
|
||
|
|
"rewards/rejected": 0.11544412752477134,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 22.0147647857666,
|
||
|
|
"kl": 19.678110122680664,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -38655453.653666146,
|
||
|
|
"logits/rejected": -38387356.24413145,
|
||
|
|
"logps/chosen": -507.5981864274571,
|
||
|
|
"logps/rejected": -364.62279929577466,
|
||
|
|
"loss": 0.5866,
|
||
|
|
"loss/base_kto": 3.7172722816467285,
|
||
|
|
"metrics/advantage_var": 231.2719268798828,
|
||
|
|
"regularization/lipschitz_norm": 1007.4017944335938,
|
||
|
|
"regularization/w1": 0.9751648306846619,
|
||
|
|
"rewards/chosen": 0.3712362447134604,
|
||
|
|
"rewards/margins": 0.257120274477985,
|
||
|
|
"rewards/rejected": 0.11411597023547536,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 18.77095603942871,
|
||
|
|
"kl": 12.41693115234375,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -35779107.36336336,
|
||
|
|
"logits/rejected": -36548648.02605863,
|
||
|
|
"logps/chosen": -489.91272522522524,
|
||
|
|
"logps/rejected": -388.24109324104234,
|
||
|
|
"loss": 0.5905,
|
||
|
|
"loss/base_kto": 3.728929281234741,
|
||
|
|
"metrics/advantage_var": 234.8322296142578,
|
||
|
|
"regularization/lipschitz_norm": 1027.6851806640625,
|
||
|
|
"regularization/w1": 0.9947992563247681,
|
||
|
|
"rewards/chosen": 0.25314079032645925,
|
||
|
|
"rewards/margins": 0.26615497175554875,
|
||
|
|
"rewards/rejected": -0.013014181429089475,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 24.464242935180664,
|
||
|
|
"kl": 22.77442741394043,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -35359588.17391305,
|
||
|
|
"logits/rejected": -38313011.52201258,
|
||
|
|
"logps/chosen": -494.44565217391306,
|
||
|
|
"logps/rejected": -355.24002555031444,
|
||
|
|
"loss": 0.5887,
|
||
|
|
"loss/base_kto": 3.7706382274627686,
|
||
|
|
"metrics/advantage_var": 218.9542236328125,
|
||
|
|
"regularization/lipschitz_norm": 969.802734375,
|
||
|
|
"regularization/w1": 0.9387690424919128,
|
||
|
|
"rewards/chosen": 0.32858901882763975,
|
||
|
|
"rewards/margins": 0.18018208438937816,
|
||
|
|
"rewards/rejected": 0.14840693443826158,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 21.17736053466797,
|
||
|
|
"kl": 19.017377853393555,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -37415138.49221184,
|
||
|
|
"logits/rejected": -39053292.73981191,
|
||
|
|
"logps/chosen": -477.3367406542056,
|
||
|
|
"logps/rejected": -378.5361726097179,
|
||
|
|
"loss": 0.5871,
|
||
|
|
"loss/base_kto": 3.7642414569854736,
|
||
|
|
"metrics/advantage_var": 217.3443603515625,
|
||
|
|
"regularization/lipschitz_norm": 963.2283325195312,
|
||
|
|
"regularization/w1": 0.9324049353599548,
|
||
|
|
"rewards/chosen": 0.3229990451135368,
|
||
|
|
"rewards/margins": 0.21354020548178348,
|
||
|
|
"rewards/rejected": 0.10945883963175328,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 22.093955993652344,
|
||
|
|
"kl": 24.88479232788086,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -37692469.72839506,
|
||
|
|
"logits/rejected": -37927268.4556962,
|
||
|
|
"logps/chosen": -482.25491898148147,
|
||
|
|
"logps/rejected": -380.12490110759495,
|
||
|
|
"loss": 0.5888,
|
||
|
|
"loss/base_kto": 3.74531626701355,
|
||
|
|
"metrics/advantage_var": 242.7454071044922,
|
||
|
|
"regularization/lipschitz_norm": 996.7890625,
|
||
|
|
"regularization/w1": 0.9648918509483337,
|
||
|
|
"rewards/chosen": 0.428317411446277,
|
||
|
|
"rewards/margins": 0.19805716603859305,
|
||
|
|
"rewards/rejected": 0.23026024540768394,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 20.8978328704834,
|
||
|
|
"kl": 16.53737449645996,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -37972866.01892745,
|
||
|
|
"logits/rejected": -39153207.47987616,
|
||
|
|
"logps/chosen": -484.7436415615142,
|
||
|
|
"logps/rejected": -412.47779605263156,
|
||
|
|
"loss": 0.6012,
|
||
|
|
"loss/base_kto": 3.7755188941955566,
|
||
|
|
"metrics/advantage_var": 272.7333984375,
|
||
|
|
"regularization/lipschitz_norm": 1068.3348388671875,
|
||
|
|
"regularization/w1": 1.034148097038269,
|
||
|
|
"rewards/chosen": 0.2477119999353066,
|
||
|
|
"rewards/margins": 0.19940300740181474,
|
||
|
|
"rewards/rejected": 0.048308992533491864,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 16.93120574951172,
|
||
|
|
"kl": 14.28690242767334,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -34974666.446909666,
|
||
|
|
"logits/rejected": -37304790.18798151,
|
||
|
|
"logps/chosen": -501.01005348652933,
|
||
|
|
"logps/rejected": -368.0087394067797,
|
||
|
|
"loss": 0.5904,
|
||
|
|
"loss/base_kto": 3.686724901199341,
|
||
|
|
"metrics/advantage_var": 277.653564453125,
|
||
|
|
"regularization/lipschitz_norm": 1070.8505859375,
|
||
|
|
"regularization/w1": 1.0365833044052124,
|
||
|
|
"rewards/chosen": 0.2573503594013099,
|
||
|
|
"rewards/margins": 0.3104527439479983,
|
||
|
|
"rewards/rejected": -0.05310238454668841,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 16.04364585876465,
|
||
|
|
"kl": 22.210905075073242,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -38007527.32530121,
|
||
|
|
"logits/rejected": -38478475.63636363,
|
||
|
|
"logps/chosen": -483.1725809487952,
|
||
|
|
"logps/rejected": -374.72752637987014,
|
||
|
|
"loss": 0.6012,
|
||
|
|
"loss/base_kto": 3.7326862812042236,
|
||
|
|
"metrics/advantage_var": 306.71075439453125,
|
||
|
|
"regularization/lipschitz_norm": 1112.4329833984375,
|
||
|
|
"regularization/w1": 1.07683527469635,
|
||
|
|
"rewards/chosen": 0.4405103936252824,
|
||
|
|
"rewards/margins": 0.22328726336199098,
|
||
|
|
"rewards/rejected": 0.2172231302632914,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 17.009994506835938,
|
||
|
|
"kl": 22.304758071899414,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -37415085.55932204,
|
||
|
|
"logits/rejected": -39361560.34231379,
|
||
|
|
"logps/chosen": -506.56346302003084,
|
||
|
|
"logps/rejected": -370.74187797147385,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.730022430419922,
|
||
|
|
"metrics/advantage_var": 254.05996704101562,
|
||
|
|
"regularization/lipschitz_norm": 1108.2991943359375,
|
||
|
|
"regularization/w1": 1.072833776473999,
|
||
|
|
"rewards/chosen": 0.3808436694975443,
|
||
|
|
"rewards/margins": 0.23417684194052912,
|
||
|
|
"rewards/rejected": 0.14666682755701516,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 23.762605667114258,
|
||
|
|
"kl": 20.390317916870117,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -37472712.22113821,
|
||
|
|
"logits/rejected": -40082581.36541353,
|
||
|
|
"logps/chosen": -503.1685975609756,
|
||
|
|
"logps/rejected": -344.64191729323306,
|
||
|
|
"loss": 0.5827,
|
||
|
|
"loss/base_kto": 3.7008392810821533,
|
||
|
|
"metrics/advantage_var": 255.104248046875,
|
||
|
|
"regularization/lipschitz_norm": 992.3699340820312,
|
||
|
|
"regularization/w1": 0.9606141448020935,
|
||
|
|
"rewards/chosen": 0.30685841630144817,
|
||
|
|
"rewards/margins": 0.25030279308202263,
|
||
|
|
"rewards/rejected": 0.05655562321942552,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 22.993144989013672,
|
||
|
|
"kl": 13.719802856445312,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -37090411.87175573,
|
||
|
|
"logits/rejected": -37467907.6864,
|
||
|
|
"logps/chosen": -468.89274809160304,
|
||
|
|
"logps/rejected": -393.59035,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.7587051391601562,
|
||
|
|
"metrics/advantage_var": 267.5472106933594,
|
||
|
|
"regularization/lipschitz_norm": 1078.6600341796875,
|
||
|
|
"regularization/w1": 1.044142723083496,
|
||
|
|
"rewards/chosen": 0.2237522270843273,
|
||
|
|
"rewards/margins": 0.22937926108090934,
|
||
|
|
"rewards/rejected": -0.005627033996582031,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 21.0300350189209,
|
||
|
|
"kl": 20.42609214782715,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -36306361.69305331,
|
||
|
|
"logits/rejected": -38323241.82753404,
|
||
|
|
"logps/chosen": -481.93755048465266,
|
||
|
|
"logps/rejected": -360.97759077155825,
|
||
|
|
"loss": 0.5868,
|
||
|
|
"loss/base_kto": 3.732846975326538,
|
||
|
|
"metrics/advantage_var": 230.20068359375,
|
||
|
|
"regularization/lipschitz_norm": 992.978515625,
|
||
|
|
"regularization/w1": 0.9612032175064087,
|
||
|
|
"rewards/chosen": 0.3177708886166574,
|
||
|
|
"rewards/margins": 0.21695634629353194,
|
||
|
|
"rewards/rejected": 0.10081454232312548,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 21.744836807250977,
|
||
|
|
"kl": 12.445245742797852,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36472679.55987055,
|
||
|
|
"logits/rejected": -37536062.64652568,
|
||
|
|
"logps/chosen": -480.65544093851133,
|
||
|
|
"logps/rejected": -357.3022092145015,
|
||
|
|
"loss": 0.5845,
|
||
|
|
"loss/base_kto": 3.6727676391601562,
|
||
|
|
"metrics/advantage_var": 261.7809753417969,
|
||
|
|
"regularization/lipschitz_norm": 1036.7802734375,
|
||
|
|
"regularization/w1": 1.0036033391952515,
|
||
|
|
"rewards/chosen": 0.21714681483395276,
|
||
|
|
"rewards/margins": 0.28658312151366977,
|
||
|
|
"rewards/rejected": -0.06943630667971701,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 19.334205627441406,
|
||
|
|
"kl": 19.236360549926758,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -37018689.76146789,
|
||
|
|
"logits/rejected": -38043680.715654954,
|
||
|
|
"logps/chosen": -485.98805428134557,
|
||
|
|
"logps/rejected": -381.8394568690096,
|
||
|
|
"loss": 0.593,
|
||
|
|
"loss/base_kto": 3.6872031688690186,
|
||
|
|
"metrics/advantage_var": 271.9364318847656,
|
||
|
|
"regularization/lipschitz_norm": 1091.694580078125,
|
||
|
|
"regularization/w1": 1.056760311126709,
|
||
|
|
"rewards/chosen": 0.309730553116638,
|
||
|
|
"rewards/margins": 0.29016388005720495,
|
||
|
|
"rewards/rejected": 0.01956667305943303,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 19.244548797607422,
|
||
|
|
"kl": 13.632672309875488,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36514600.59546926,
|
||
|
|
"logits/rejected": -37751418.19939577,
|
||
|
|
"logps/chosen": -495.4087783171521,
|
||
|
|
"logps/rejected": -384.47288047583083,
|
||
|
|
"loss": 0.5902,
|
||
|
|
"loss/base_kto": 3.7466392517089844,
|
||
|
|
"metrics/advantage_var": 240.673828125,
|
||
|
|
"regularization/lipschitz_norm": 1007.3256225585938,
|
||
|
|
"regularization/w1": 0.9750910997390747,
|
||
|
|
"rewards/chosen": 0.2429255019499646,
|
||
|
|
"rewards/margins": 0.2508964667505198,
|
||
|
|
"rewards/rejected": -0.00797096480055518,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 28.49403953552246,
|
||
|
|
"kl": 6.630456447601318,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -36511871.805175036,
|
||
|
|
"logits/rejected": -38109210.298555374,
|
||
|
|
"logps/chosen": -464.95928462709287,
|
||
|
|
"logps/rejected": -373.99934791332265,
|
||
|
|
"loss": 0.6031,
|
||
|
|
"loss/base_kto": 3.8054044246673584,
|
||
|
|
"metrics/advantage_var": 254.1283721923828,
|
||
|
|
"regularization/lipschitz_norm": 1052.9443359375,
|
||
|
|
"regularization/w1": 1.0192501544952393,
|
||
|
|
"rewards/chosen": 0.03811651314048825,
|
||
|
|
"rewards/margins": 0.19720065563866992,
|
||
|
|
"rewards/rejected": -0.15908414249818167,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 21.918201446533203,
|
||
|
|
"kl": 11.34065055847168,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36806082.05819295,
|
||
|
|
"logits/rejected": -37579290.94736842,
|
||
|
|
"logps/chosen": -496.1367725880551,
|
||
|
|
"logps/rejected": -365.33378189792666,
|
||
|
|
"loss": 0.5917,
|
||
|
|
"loss/base_kto": 3.709606885910034,
|
||
|
|
"metrics/advantage_var": 249.86502075195312,
|
||
|
|
"regularization/lipschitz_norm": 1057.7276611328125,
|
||
|
|
"regularization/w1": 1.0238803625106812,
|
||
|
|
"rewards/chosen": 0.2249710760656944,
|
||
|
|
"rewards/margins": 0.2674478974297123,
|
||
|
|
"rewards/rejected": -0.04247682136401795,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 25.505876541137695,
|
||
|
|
"kl": 11.668891906738281,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -37083213.283018865,
|
||
|
|
"logits/rejected": -37264530.28571428,
|
||
|
|
"logps/chosen": -485.0509040880503,
|
||
|
|
"logps/rejected": -365.58503979037266,
|
||
|
|
"loss": 0.5885,
|
||
|
|
"loss/base_kto": 3.721874237060547,
|
||
|
|
"metrics/advantage_var": 244.218994140625,
|
||
|
|
"regularization/lipschitz_norm": 1018.9505004882812,
|
||
|
|
"regularization/w1": 0.9863441586494446,
|
||
|
|
"rewards/chosen": 0.19097203428640305,
|
||
|
|
"rewards/margins": 0.2519630727332441,
|
||
|
|
"rewards/rejected": -0.060991038446841034,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 20.59295654296875,
|
||
|
|
"kl": 15.112597465515137,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -37952208.285251215,
|
||
|
|
"logits/rejected": -38671818.71493213,
|
||
|
|
"logps/chosen": -489.17696515397085,
|
||
|
|
"logps/rejected": -388.3527762066365,
|
||
|
|
"loss": 0.5913,
|
||
|
|
"loss/base_kto": 3.7084343433380127,
|
||
|
|
"metrics/advantage_var": 286.5562744140625,
|
||
|
|
"regularization/lipschitz_norm": 1055.5869140625,
|
||
|
|
"regularization/w1": 1.02180814743042,
|
||
|
|
"rewards/chosen": 0.2032725923065235,
|
||
|
|
"rewards/margins": 0.26045843708463906,
|
||
|
|
"rewards/rejected": -0.05718584477811557,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 18.425968170166016,
|
||
|
|
"kl": 10.225451469421387,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -35257863.87692308,
|
||
|
|
"logits/rejected": -38423239.92380952,
|
||
|
|
"logps/chosen": -462.68048076923077,
|
||
|
|
"logps/rejected": -381.8985863095238,
|
||
|
|
"loss": 0.5899,
|
||
|
|
"loss/base_kto": 3.7217705249786377,
|
||
|
|
"metrics/advantage_var": 243.5096435546875,
|
||
|
|
"regularization/lipschitz_norm": 1030.2635498046875,
|
||
|
|
"regularization/w1": 0.997295081615448,
|
||
|
|
"rewards/chosen": 0.14774301382211538,
|
||
|
|
"rewards/margins": 0.24367004338638248,
|
||
|
|
"rewards/rejected": -0.09592702956426712,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 14.970965385437012,
|
||
|
|
"kl": 10.264970779418945,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36357974.3803681,
|
||
|
|
"logits/rejected": -37892461.24840765,
|
||
|
|
"logps/chosen": -497.34633819018404,
|
||
|
|
"logps/rejected": -362.88460390127386,
|
||
|
|
"loss": 0.6052,
|
||
|
|
"loss/base_kto": 3.768388509750366,
|
||
|
|
"metrics/advantage_var": 271.6504821777344,
|
||
|
|
"regularization/lipschitz_norm": 1108.8814697265625,
|
||
|
|
"regularization/w1": 1.0733972787857056,
|
||
|
|
"rewards/chosen": 0.13551885218708062,
|
||
|
|
"rewards/margins": 0.2049240495250148,
|
||
|
|
"rewards/rejected": -0.06940519733793417,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 19.641616821289062,
|
||
|
|
"kl": 17.239547729492188,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -35284543.89627229,
|
||
|
|
"logits/rejected": -36789311.32428356,
|
||
|
|
"logps/chosen": -499.28307333873585,
|
||
|
|
"logps/rejected": -373.6219834087481,
|
||
|
|
"loss": 0.6078,
|
||
|
|
"loss/base_kto": 3.773378849029541,
|
||
|
|
"metrics/advantage_var": 287.3152770996094,
|
||
|
|
"regularization/lipschitz_norm": 1124.9940185546875,
|
||
|
|
"regularization/w1": 1.0889942646026611,
|
||
|
|
"rewards/chosen": 0.2207545152166481,
|
||
|
|
"rewards/margins": 0.20434218874337173,
|
||
|
|
"rewards/rejected": 0.01641232647327636,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 23.141918182373047,
|
||
|
|
"kl": 10.620841026306152,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -35900389.574193545,
|
||
|
|
"logits/rejected": -36876991.416413374,
|
||
|
|
"logps/chosen": -517.7074092741935,
|
||
|
|
"logps/rejected": -376.32734137537994,
|
||
|
|
"loss": 0.5987,
|
||
|
|
"loss/base_kto": 3.5631072521209717,
|
||
|
|
"metrics/advantage_var": 381.3092346191406,
|
||
|
|
"regularization/lipschitz_norm": 1266.8702392578125,
|
||
|
|
"regularization/w1": 1.2263303995132446,
|
||
|
|
"rewards/chosen": 0.16041895958685107,
|
||
|
|
"rewards/margins": 0.44302277129728995,
|
||
|
|
"rewards/rejected": -0.28260381171043886,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 26.886734008789062,
|
||
|
|
"kl": 7.9778265953063965,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -36258746.658267714,
|
||
|
|
"logits/rejected": -36793936.17364341,
|
||
|
|
"logps/chosen": -481.85147637795274,
|
||
|
|
"logps/rejected": -378.1058624031008,
|
||
|
|
"loss": 0.6187,
|
||
|
|
"loss/base_kto": 3.3150124549865723,
|
||
|
|
"metrics/advantage_var": 659.8348999023438,
|
||
|
|
"regularization/lipschitz_norm": 1688.483642578125,
|
||
|
|
"regularization/w1": 1.6344521045684814,
|
||
|
|
"rewards/chosen": 0.32000405619463584,
|
||
|
|
"rewards/margins": 0.7908002006653239,
|
||
|
|
"rewards/rejected": -0.47079614447068796,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 18.536203384399414,
|
||
|
|
"kl": 10.486624717712402,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -35131913.30909091,
|
||
|
|
"logits/rejected": -36720418.68387097,
|
||
|
|
"logps/chosen": -481.3445075757576,
|
||
|
|
"logps/rejected": -368.8100050403226,
|
||
|
|
"loss": 0.607,
|
||
|
|
"loss/base_kto": 3.3354427814483643,
|
||
|
|
"metrics/advantage_var": 581.4777221679688,
|
||
|
|
"regularization/lipschitz_norm": 1570.6834716796875,
|
||
|
|
"regularization/w1": 1.5204215049743652,
|
||
|
|
"rewards/chosen": 0.4108243537671638,
|
||
|
|
"rewards/margins": 0.711322922394306,
|
||
|
|
"rewards/rejected": -0.30049856862714214,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 17.286394119262695,
|
||
|
|
"kl": 12.479207038879395,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -35551176.47208931,
|
||
|
|
"logits/rejected": -36891753.06584992,
|
||
|
|
"logps/chosen": -479.0072767145136,
|
||
|
|
"logps/rejected": -420.85356049004594,
|
||
|
|
"loss": 0.6028,
|
||
|
|
"loss/base_kto": 3.236600637435913,
|
||
|
|
"metrics/advantage_var": 646.4462280273438,
|
||
|
|
"regularization/lipschitz_norm": 1638.5428466796875,
|
||
|
|
"regularization/w1": 1.5861095190048218,
|
||
|
|
"rewards/chosen": 0.4569494804126794,
|
||
|
|
"rewards/margins": 0.8522189940905365,
|
||
|
|
"rewards/rejected": -0.395269513677857,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 18.596973419189453,
|
||
|
|
"kl": 10.5941801071167,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -36412026.452648476,
|
||
|
|
"logits/rejected": -37210289.680365294,
|
||
|
|
"logps/chosen": -460.2596308186196,
|
||
|
|
"logps/rejected": -381.8862014840183,
|
||
|
|
"loss": 0.6332,
|
||
|
|
"loss/base_kto": 3.3203790187835693,
|
||
|
|
"metrics/advantage_var": 744.6414184570312,
|
||
|
|
"regularization/lipschitz_norm": 1803.1883544921875,
|
||
|
|
"regularization/w1": 1.7454862594604492,
|
||
|
|
"rewards/chosen": 0.42800364486669845,
|
||
|
|
"rewards/margins": 0.8059915573045671,
|
||
|
|
"rewards/rejected": -0.3779879124378686,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 25.376361846923828,
|
||
|
|
"kl": 10.37247085571289,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -36870867.91540785,
|
||
|
|
"logits/rejected": -37704650.97734628,
|
||
|
|
"logps/chosen": -477.55173716012087,
|
||
|
|
"logps/rejected": -379.4901395631068,
|
||
|
|
"loss": 0.6147,
|
||
|
|
"loss/base_kto": 3.3837063312530518,
|
||
|
|
"metrics/advantage_var": 606.6046752929688,
|
||
|
|
"regularization/lipschitz_norm": 1584.3231201171875,
|
||
|
|
"regularization/w1": 1.5336246490478516,
|
||
|
|
"rewards/chosen": 0.25861393216873585,
|
||
|
|
"rewards/margins": 0.7301805367336125,
|
||
|
|
"rewards/rejected": -0.47156660456487665,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 17.33742904663086,
|
||
|
|
"kl": 24.371213912963867,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -36366941.8798151,
|
||
|
|
"logits/rejected": -37636676.96988907,
|
||
|
|
"logps/chosen": -478.2693567026194,
|
||
|
|
"logps/rejected": -365.2568343898574,
|
||
|
|
"loss": 0.593,
|
||
|
|
"loss/base_kto": 3.240446090698242,
|
||
|
|
"metrics/advantage_var": 607.9482421875,
|
||
|
|
"regularization/lipschitz_norm": 1553.604736328125,
|
||
|
|
"regularization/w1": 1.5038893222808838,
|
||
|
|
"rewards/chosen": 0.7234842215186826,
|
||
|
|
"rewards/margins": 0.8010088329056181,
|
||
|
|
"rewards/rejected": -0.07752461138693542,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 21.46354866027832,
|
||
|
|
"kl": 20.619953155517578,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -37012783.82417583,
|
||
|
|
"logits/rejected": -38748924.41679627,
|
||
|
|
"logps/chosen": -483.8767660910518,
|
||
|
|
"logps/rejected": -370.6149883359254,
|
||
|
|
"loss": 0.613,
|
||
|
|
"loss/base_kto": 3.2702362537384033,
|
||
|
|
"metrics/advantage_var": 731.5919799804688,
|
||
|
|
"regularization/lipschitz_norm": 1687.517578125,
|
||
|
|
"regularization/w1": 1.6335169076919556,
|
||
|
|
"rewards/chosen": 0.6287614711599784,
|
||
|
|
"rewards/margins": 0.8008188647101095,
|
||
|
|
"rewards/rejected": -0.17205739355013122,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 27.949670791625977,
|
||
|
|
"kl": 23.042285919189453,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35671269.35015773,
|
||
|
|
"logits/rejected": -36058314.89783282,
|
||
|
|
"logps/chosen": -489.92044558359623,
|
||
|
|
"logps/rejected": -373.0068933823529,
|
||
|
|
"loss": 0.6269,
|
||
|
|
"loss/base_kto": 3.339374542236328,
|
||
|
|
"metrics/advantage_var": 702.3568725585938,
|
||
|
|
"regularization/lipschitz_norm": 1730.974609375,
|
||
|
|
"regularization/w1": 1.6755837202072144,
|
||
|
|
"rewards/chosen": 0.5970071016425966,
|
||
|
|
"rewards/margins": 0.699064118682554,
|
||
|
|
"rewards/rejected": -0.10205701703995743,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 16.850645065307617,
|
||
|
|
"kl": 19.39977264404297,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -36006448.87636933,
|
||
|
|
"logits/rejected": -36771542.86427457,
|
||
|
|
"logps/chosen": -447.0553599374022,
|
||
|
|
"logps/rejected": -370.2153129875195,
|
||
|
|
"loss": 0.5865,
|
||
|
|
"loss/base_kto": 3.266871690750122,
|
||
|
|
"metrics/advantage_var": 522.5993041992188,
|
||
|
|
"regularization/lipschitz_norm": 1472.3348388671875,
|
||
|
|
"regularization/w1": 1.425220012664795,
|
||
|
|
"rewards/chosen": 0.5224020037106318,
|
||
|
|
"rewards/margins": 0.7634551162941196,
|
||
|
|
"rewards/rejected": -0.24105311258348772,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 22.18784523010254,
|
||
|
|
"kl": 16.881107330322266,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -35078514.87179487,
|
||
|
|
"logits/rejected": -36275562.146341465,
|
||
|
|
"logps/chosen": -498.1405248397436,
|
||
|
|
"logps/rejected": -367.7579554115854,
|
||
|
|
"loss": 0.6019,
|
||
|
|
"loss/base_kto": 3.342583417892456,
|
||
|
|
"metrics/advantage_var": 527.4892578125,
|
||
|
|
"regularization/lipschitz_norm": 1521.4276123046875,
|
||
|
|
"regularization/w1": 1.472741961479187,
|
||
|
|
"rewards/chosen": 0.4546708327073317,
|
||
|
|
"rewards/margins": 0.7038324134211156,
|
||
|
|
"rewards/rejected": -0.24916158071378383,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 17.10660743713379,
|
||
|
|
"kl": 18.817310333251953,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -36048223.52238806,
|
||
|
|
"logits/rejected": -37401690.64918033,
|
||
|
|
"logps/chosen": -495.32285447761194,
|
||
|
|
"logps/rejected": -388.59574795081966,
|
||
|
|
"loss": 0.6225,
|
||
|
|
"loss/base_kto": 3.279219150543213,
|
||
|
|
"metrics/advantage_var": 737.9172973632812,
|
||
|
|
"regularization/lipschitz_norm": 1756.8636474609375,
|
||
|
|
"regularization/w1": 1.7006438970565796,
|
||
|
|
"rewards/chosen": 0.609760887943097,
|
||
|
|
"rewards/margins": 0.7567953172743919,
|
||
|
|
"rewards/rejected": -0.14703442933129482,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 18.01519203186035,
|
||
|
|
"kl": 8.799586296081543,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -34808334.266253866,
|
||
|
|
"logits/rejected": -36906529.91798107,
|
||
|
|
"logps/chosen": -470.0831075851393,
|
||
|
|
"logps/rejected": -366.04475552050474,
|
||
|
|
"loss": 0.6028,
|
||
|
|
"loss/base_kto": 3.363507032394409,
|
||
|
|
"metrics/advantage_var": 511.4743347167969,
|
||
|
|
"regularization/lipschitz_norm": 1507.081298828125,
|
||
|
|
"regularization/w1": 1.4588545560836792,
|
||
|
|
"rewards/chosen": 0.36784457274634774,
|
||
|
|
"rewards/margins": 0.6944655471215941,
|
||
|
|
"rewards/rejected": -0.32662097437524645,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 17.3218936920166,
|
||
|
|
"kl": 15.690451622009277,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -35773232.096969694,
|
||
|
|
"logits/rejected": -35636138.11612903,
|
||
|
|
"logps/chosen": -501.4535984848485,
|
||
|
|
"logps/rejected": -365.37310987903226,
|
||
|
|
"loss": 0.6102,
|
||
|
|
"loss/base_kto": 3.2086281776428223,
|
||
|
|
"metrics/advantage_var": 698.2100830078125,
|
||
|
|
"regularization/lipschitz_norm": 1728.182861328125,
|
||
|
|
"regularization/w1": 1.672881007194519,
|
||
|
|
"rewards/chosen": 0.6377998120857008,
|
||
|
|
"rewards/margins": 0.8835186880937653,
|
||
|
|
"rewards/rejected": -0.24571887600806452,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 17.114648818969727,
|
||
|
|
"kl": 23.868871688842773,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -35142348.96075353,
|
||
|
|
"logits/rejected": -37201375.353032656,
|
||
|
|
"logps/chosen": -504.3203492935636,
|
||
|
|
"logps/rejected": -371.99927099533437,
|
||
|
|
"loss": 0.6056,
|
||
|
|
"loss/base_kto": 3.1933350563049316,
|
||
|
|
"metrics/advantage_var": 700.6327514648438,
|
||
|
|
"regularization/lipschitz_norm": 1706.2183837890625,
|
||
|
|
"regularization/w1": 1.6516193151474,
|
||
|
|
"rewards/chosen": 0.7034476143973214,
|
||
|
|
"rewards/margins": 0.9059251244135418,
|
||
|
|
"rewards/rejected": -0.20247751001622036,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 18.362234115600586,
|
||
|
|
"kl": 20.849035263061523,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34272197.44049459,
|
||
|
|
"logits/rejected": -35951044.95418642,
|
||
|
|
"logps/chosen": -470.51294435857807,
|
||
|
|
"logps/rejected": -372.9975809636651,
|
||
|
|
"loss": 0.6039,
|
||
|
|
"loss/base_kto": 3.3228371143341064,
|
||
|
|
"metrics/advantage_var": 573.0205688476562,
|
||
|
|
"regularization/lipschitz_norm": 1558.5523681640625,
|
||
|
|
"regularization/w1": 1.5086787939071655,
|
||
|
|
"rewards/chosen": 0.530442911469402,
|
||
|
|
"rewards/margins": 0.6922132665493783,
|
||
|
|
"rewards/rejected": -0.1617703550799763,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 23.748563766479492,
|
||
|
|
"kl": 17.160327911376953,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34102851.62264151,
|
||
|
|
"logits/rejected": -35446984.347826086,
|
||
|
|
"logps/chosen": -465.8104363207547,
|
||
|
|
"logps/rejected": -373.1613451086956,
|
||
|
|
"loss": 0.6057,
|
||
|
|
"loss/base_kto": 3.3180763721466064,
|
||
|
|
"metrics/advantage_var": 569.592041015625,
|
||
|
|
"regularization/lipschitz_norm": 1578.3482666015625,
|
||
|
|
"regularization/w1": 1.5278409719467163,
|
||
|
|
"rewards/chosen": 0.5636146593393769,
|
||
|
|
"rewards/margins": 0.7727055378437806,
|
||
|
|
"rewards/rejected": -0.20909087850440364,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 23.817169189453125,
|
||
|
|
"kl": 13.761589050292969,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -33891055.47086614,
|
||
|
|
"logits/rejected": -35839542.77209302,
|
||
|
|
"logps/chosen": -475.5876968503937,
|
||
|
|
"logps/rejected": -379.225,
|
||
|
|
"loss": 0.5958,
|
||
|
|
"loss/base_kto": 3.3010895252227783,
|
||
|
|
"metrics/advantage_var": 584.8517456054688,
|
||
|
|
"regularization/lipschitz_norm": 1513.895751953125,
|
||
|
|
"regularization/w1": 1.4654511213302612,
|
||
|
|
"rewards/chosen": 0.46517670398622046,
|
||
|
|
"rewards/margins": 0.74915045697647,
|
||
|
|
"rewards/rejected": -0.2839737529902495,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 16.948810577392578,
|
||
|
|
"kl": 7.98611307144165,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -34369874.56726094,
|
||
|
|
"logits/rejected": -37922723.330316745,
|
||
|
|
"logps/chosen": -480.9890599675851,
|
||
|
|
"logps/rejected": -365.08790535444945,
|
||
|
|
"loss": 0.6066,
|
||
|
|
"loss/base_kto": 3.2714080810546875,
|
||
|
|
"metrics/advantage_var": 632.8931274414062,
|
||
|
|
"regularization/lipschitz_norm": 1633.6640625,
|
||
|
|
"regularization/w1": 1.5813868045806885,
|
||
|
|
"rewards/chosen": 0.37567811344585694,
|
||
|
|
"rewards/margins": 0.805177997083206,
|
||
|
|
"rewards/rejected": -0.42949988363734914,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 18.99588394165039,
|
||
|
|
"kl": 8.318107604980469,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -34767724.291734196,
|
||
|
|
"logits/rejected": -35584108.11463047,
|
||
|
|
"logps/chosen": -490.5239566450567,
|
||
|
|
"logps/rejected": -373.4036104826546,
|
||
|
|
"loss": 0.6046,
|
||
|
|
"loss/base_kto": 3.2944839000701904,
|
||
|
|
"metrics/advantage_var": 566.6473388671875,
|
||
|
|
"regularization/lipschitz_norm": 1593.2259521484375,
|
||
|
|
"regularization/w1": 1.5422428846359253,
|
||
|
|
"rewards/chosen": 0.3456378361780794,
|
||
|
|
"rewards/margins": 0.7647938718645896,
|
||
|
|
"rewards/rejected": -0.41915603568651016,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 18.505903244018555,
|
||
|
|
"kl": 8.197784423828125,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34011293.055214725,
|
||
|
|
"logits/rejected": -36369544.968152866,
|
||
|
|
"logps/chosen": -490.1889378834356,
|
||
|
|
"logps/rejected": -384.5946705812102,
|
||
|
|
"loss": 0.6172,
|
||
|
|
"loss/base_kto": 3.3246331214904785,
|
||
|
|
"metrics/advantage_var": 656.5321655273438,
|
||
|
|
"regularization/lipschitz_norm": 1666.1669921875,
|
||
|
|
"regularization/w1": 1.6128495931625366,
|
||
|
|
"rewards/chosen": 0.3669387255709595,
|
||
|
|
"rewards/margins": 0.7547161634006272,
|
||
|
|
"rewards/rejected": -0.3877774378296676,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 16.069843292236328,
|
||
|
|
"kl": 3.699871063232422,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -33936347.14241002,
|
||
|
|
"logits/rejected": -34886366.85179407,
|
||
|
|
"logps/chosen": -475.2531298904538,
|
||
|
|
"logps/rejected": -387.1373098673947,
|
||
|
|
"loss": 0.6161,
|
||
|
|
"loss/base_kto": 3.3539459705352783,
|
||
|
|
"metrics/advantage_var": 633.36083984375,
|
||
|
|
"regularization/lipschitz_norm": 1627.1385498046875,
|
||
|
|
"regularization/w1": 1.5750700235366821,
|
||
|
|
"rewards/chosen": 0.13495619270909748,
|
||
|
|
"rewards/margins": 0.7070479484496397,
|
||
|
|
"rewards/rejected": -0.5720917557405422,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 18.5937557220459,
|
||
|
|
"kl": 13.287925720214844,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -34599509.60252366,
|
||
|
|
"logits/rejected": -34751373.86996904,
|
||
|
|
"logps/chosen": -473.21520110410097,
|
||
|
|
"logps/rejected": -411.38694852941177,
|
||
|
|
"loss": 0.6095,
|
||
|
|
"loss/base_kto": 3.303356170654297,
|
||
|
|
"metrics/advantage_var": 738.0794067382812,
|
||
|
|
"regularization/lipschitz_norm": 1624.4622802734375,
|
||
|
|
"regularization/w1": 1.572479486465454,
|
||
|
|
"rewards/chosen": 0.41713139386583203,
|
||
|
|
"rewards/margins": 0.786764929717467,
|
||
|
|
"rewards/rejected": -0.36963353585163505,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 14.898553848266602,
|
||
|
|
"kl": 9.77316951751709,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -34896962.22885572,
|
||
|
|
"logits/rejected": -36254104.389955685,
|
||
|
|
"logps/chosen": -495.8976471807629,
|
||
|
|
"logps/rejected": -352.47267355982274,
|
||
|
|
"loss": 0.6091,
|
||
|
|
"loss/base_kto": 3.2609596252441406,
|
||
|
|
"metrics/advantage_var": 734.3327026367188,
|
||
|
|
"regularization/lipschitz_norm": 1664.761962890625,
|
||
|
|
"regularization/w1": 1.6114895343780518,
|
||
|
|
"rewards/chosen": 0.3713784795100021,
|
||
|
|
"rewards/margins": 0.8420873744509179,
|
||
|
|
"rewards/rejected": -0.4707088949409158,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 17.896150588989258,
|
||
|
|
"kl": 6.074289798736572,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -33776399.15022762,
|
||
|
|
"logits/rejected": -33075975.00805153,
|
||
|
|
"logps/chosen": -508.469034522003,
|
||
|
|
"logps/rejected": -412.82168377616745,
|
||
|
|
"loss": 0.6019,
|
||
|
|
"loss/base_kto": 3.3112874031066895,
|
||
|
|
"metrics/advantage_var": 567.4923706054688,
|
||
|
|
"regularization/lipschitz_norm": 1553.6488037109375,
|
||
|
|
"regularization/w1": 1.503931999206543,
|
||
|
|
"rewards/chosen": 0.2563087104484778,
|
||
|
|
"rewards/margins": 0.7786207948983319,
|
||
|
|
"rewards/rejected": -0.5223120844498541,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 17.405357360839844,
|
||
|
|
"kl": 8.466923713684082,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -34845189.66192733,
|
||
|
|
"logits/rejected": -35608800.741885625,
|
||
|
|
"logps/chosen": -495.66903633491313,
|
||
|
|
"logps/rejected": -385.2935181607419,
|
||
|
|
"loss": 0.6216,
|
||
|
|
"loss/base_kto": 3.321855306625366,
|
||
|
|
"metrics/advantage_var": 709.4938354492188,
|
||
|
|
"regularization/lipschitz_norm": 1705.716796875,
|
||
|
|
"regularization/w1": 1.6511337757110596,
|
||
|
|
"rewards/chosen": 0.3443122851716775,
|
||
|
|
"rewards/margins": 0.7471894560233245,
|
||
|
|
"rewards/rejected": -0.40287717085164704,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 15.573775291442871,
|
||
|
|
"kl": 7.134391784667969,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -33920080.67878788,
|
||
|
|
"logits/rejected": -36426282.94193549,
|
||
|
|
"logps/chosen": -500.8137784090909,
|
||
|
|
"logps/rejected": -360.63009072580644,
|
||
|
|
"loss": 0.6084,
|
||
|
|
"loss/base_kto": 3.3668205738067627,
|
||
|
|
"metrics/advantage_var": 544.8922729492188,
|
||
|
|
"regularization/lipschitz_norm": 1549.573486328125,
|
||
|
|
"regularization/w1": 1.499987006187439,
|
||
|
|
"rewards/chosen": 0.2990838253136837,
|
||
|
|
"rewards/margins": 0.704384433302595,
|
||
|
|
"rewards/rejected": -0.40530060798891127,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 19.988651275634766,
|
||
|
|
"kl": 19.997989654541016,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -31784119.87616099,
|
||
|
|
"logits/rejected": -35308534.30914827,
|
||
|
|
"logps/chosen": -502.8266253869969,
|
||
|
|
"logps/rejected": -358.9458300473186,
|
||
|
|
"loss": 0.6203,
|
||
|
|
"loss/base_kto": 3.3223488330841064,
|
||
|
|
"metrics/advantage_var": 1186.4998779296875,
|
||
|
|
"regularization/lipschitz_norm": 1694.468017578125,
|
||
|
|
"regularization/w1": 1.6402450799942017,
|
||
|
|
"rewards/chosen": 0.5360499520788989,
|
||
|
|
"rewards/margins": 0.769386824937855,
|
||
|
|
"rewards/rejected": -0.23333687285895605,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 21.27825927734375,
|
||
|
|
"kl": 14.778144836425781,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -33192780.406153847,
|
||
|
|
"logits/rejected": -35539620.16507936,
|
||
|
|
"logps/chosen": -487.72826923076923,
|
||
|
|
"logps/rejected": -365.61153273809526,
|
||
|
|
"loss": 0.5938,
|
||
|
|
"loss/base_kto": 3.2722222805023193,
|
||
|
|
"metrics/advantage_var": 570.6756591796875,
|
||
|
|
"regularization/lipschitz_norm": 1527.1158447265625,
|
||
|
|
"regularization/w1": 1.478248119354248,
|
||
|
|
"rewards/chosen": 0.5232102614182692,
|
||
|
|
"rewards/margins": 0.751742186680236,
|
||
|
|
"rewards/rejected": -0.22853192526196678,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 18.707075119018555,
|
||
|
|
"kl": 14.053110122680664,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -34841998.31181102,
|
||
|
|
"logits/rejected": -36095298.28217054,
|
||
|
|
"logps/chosen": -475.1556594488189,
|
||
|
|
"logps/rejected": -374.0540697674419,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.334726333618164,
|
||
|
|
"metrics/advantage_var": 629.5460815429688,
|
||
|
|
"regularization/lipschitz_norm": 1516.635986328125,
|
||
|
|
"regularization/w1": 1.4681035280227661,
|
||
|
|
"rewards/chosen": 0.48633317121370573,
|
||
|
|
"rewards/margins": 0.7661481353307116,
|
||
|
|
"rewards/rejected": -0.2798149641170058,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 22.795734405517578,
|
||
|
|
"kl": 10.226655960083008,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -34476848.64797508,
|
||
|
|
"logits/rejected": -34940799.59874608,
|
||
|
|
"logps/chosen": -486.3995813862928,
|
||
|
|
"logps/rejected": -359.81235305642633,
|
||
|
|
"loss": 0.5896,
|
||
|
|
"loss/base_kto": 3.316878080368042,
|
||
|
|
"metrics/advantage_var": 529.7022705078125,
|
||
|
|
"regularization/lipschitz_norm": 1446.478759765625,
|
||
|
|
"regularization/w1": 1.4001914262771606,
|
||
|
|
"rewards/chosen": 0.39251145692629236,
|
||
|
|
"rewards/margins": 0.7439681498999404,
|
||
|
|
"rewards/rejected": -0.3514566929736481,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 17.954349517822266,
|
||
|
|
"kl": 17.41472816467285,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -35201938.6407767,
|
||
|
|
"logits/rejected": -35623357.486404836,
|
||
|
|
"logps/chosen": -483.64370954692555,
|
||
|
|
"logps/rejected": -387.6900490936556,
|
||
|
|
"loss": 0.6029,
|
||
|
|
"loss/base_kto": 3.2610299587249756,
|
||
|
|
"metrics/advantage_var": 612.34765625,
|
||
|
|
"regularization/lipschitz_norm": 1613.9171142578125,
|
||
|
|
"regularization/w1": 1.5622718334197998,
|
||
|
|
"rewards/chosen": 0.5058932011181483,
|
||
|
|
"rewards/margins": 0.8012901269377527,
|
||
|
|
"rewards/rejected": -0.2953969258196044,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 17.581260681152344,
|
||
|
|
"kl": 11.968046188354492,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -33651710.41731066,
|
||
|
|
"logits/rejected": -33495272.9478673,
|
||
|
|
"logps/chosen": -456.492658423493,
|
||
|
|
"logps/rejected": -381.71936710110583,
|
||
|
|
"loss": 0.595,
|
||
|
|
"loss/base_kto": 3.2871201038360596,
|
||
|
|
"metrics/advantage_var": 583.8927612304688,
|
||
|
|
"regularization/lipschitz_norm": 1521.3782958984375,
|
||
|
|
"regularization/w1": 1.4726942777633667,
|
||
|
|
"rewards/chosen": 0.4107969304693537,
|
||
|
|
"rewards/margins": 0.7617834051558663,
|
||
|
|
"rewards/rejected": -0.3509864746865126,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 20.793994903564453,
|
||
|
|
"kl": 14.662152290344238,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -35222220.96075353,
|
||
|
|
"logits/rejected": -36583083.197511666,
|
||
|
|
"logps/chosen": -492.78120094191524,
|
||
|
|
"logps/rejected": -378.14959175738727,
|
||
|
|
"loss": 0.6018,
|
||
|
|
"loss/base_kto": 3.297760486602783,
|
||
|
|
"metrics/advantage_var": 564.8939208984375,
|
||
|
|
"regularization/lipschitz_norm": 1567.152587890625,
|
||
|
|
"regularization/w1": 1.5170036554336548,
|
||
|
|
"rewards/chosen": 0.4784900419753238,
|
||
|
|
"rewards/margins": 0.7377271860539834,
|
||
|
|
"rewards/rejected": -0.2592371440786596,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 18.106521606445312,
|
||
|
|
"kl": 17.453224182128906,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34577372.8,
|
||
|
|
"logits/rejected": -35849900.8,
|
||
|
|
"logps/chosen": -478.12919921875,
|
||
|
|
"logps/rejected": -374.3378173828125,
|
||
|
|
"loss": 0.6005,
|
||
|
|
"loss/base_kto": 3.2170441150665283,
|
||
|
|
"metrics/advantage_var": 673.6275634765625,
|
||
|
|
"regularization/lipschitz_norm": 1639.2894287109375,
|
||
|
|
"regularization/w1": 1.5868322849273682,
|
||
|
|
"rewards/chosen": 0.5354191780090332,
|
||
|
|
"rewards/margins": 0.8580185413360595,
|
||
|
|
"rewards/rejected": -0.3225993633270264,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 19.869972229003906,
|
||
|
|
"kl": 19.689435958862305,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -35870056.2962963,
|
||
|
|
"logits/rejected": -35619895.088607594,
|
||
|
|
"logps/chosen": -462.14351851851853,
|
||
|
|
"logps/rejected": -369.37663172468353,
|
||
|
|
"loss": 0.6031,
|
||
|
|
"loss/base_kto": 3.3086509704589844,
|
||
|
|
"metrics/advantage_var": 570.68212890625,
|
||
|
|
"regularization/lipschitz_norm": 1566.245361328125,
|
||
|
|
"regularization/w1": 1.5161254405975342,
|
||
|
|
"rewards/chosen": 0.5847580521195023,
|
||
|
|
"rewards/margins": 0.7637615986290639,
|
||
|
|
"rewards/rejected": -0.17900354650956166,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 17.84902000427246,
|
||
|
|
"kl": 20.60800552368164,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -33869262.048780486,
|
||
|
|
"logits/rejected": -35129416.20512821,
|
||
|
|
"logps/chosen": -472.5958936737805,
|
||
|
|
"logps/rejected": -374.59900841346155,
|
||
|
|
"loss": 0.6061,
|
||
|
|
"loss/base_kto": 3.276806354522705,
|
||
|
|
"metrics/advantage_var": 604.2709350585938,
|
||
|
|
"regularization/lipschitz_norm": 1623.892822265625,
|
||
|
|
"regularization/w1": 1.5719282627105713,
|
||
|
|
"rewards/chosen": 0.6462965244200172,
|
||
|
|
"rewards/margins": 0.810724391722545,
|
||
|
|
"rewards/rejected": -0.16442786730252779,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 16.47085189819336,
|
||
|
|
"kl": 17.133132934570312,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -34565411.8647343,
|
||
|
|
"logits/rejected": -37085233.72382397,
|
||
|
|
"logps/chosen": -497.7308776167472,
|
||
|
|
"logps/rejected": -355.6858402883156,
|
||
|
|
"loss": 0.5882,
|
||
|
|
"loss/base_kto": 3.306173324584961,
|
||
|
|
"metrics/advantage_var": 500.00372314453125,
|
||
|
|
"regularization/lipschitz_norm": 1446.0572509765625,
|
||
|
|
"regularization/w1": 1.3997834920883179,
|
||
|
|
"rewards/chosen": 0.4956057636058273,
|
||
|
|
"rewards/margins": 0.7162988164157784,
|
||
|
|
"rewards/rejected": -0.22069305280995116,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 16.752580642700195,
|
||
|
|
"kl": 14.875192642211914,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -33914434.37037037,
|
||
|
|
"logits/rejected": -35248309.468354434,
|
||
|
|
"logps/chosen": -442.65928819444446,
|
||
|
|
"logps/rejected": -374.595332278481,
|
||
|
|
"loss": 0.597,
|
||
|
|
"loss/base_kto": 3.3261163234710693,
|
||
|
|
"metrics/advantage_var": 571.9568481445312,
|
||
|
|
"regularization/lipschitz_norm": 1497.6256103515625,
|
||
|
|
"regularization/w1": 1.4497016668319702,
|
||
|
|
"rewards/chosen": 0.4288687529387297,
|
||
|
|
"rewards/margins": 0.7263318923399362,
|
||
|
|
"rewards/rejected": -0.2974631394012065,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 14.579239845275879,
|
||
|
|
"kl": 16.715412139892578,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -35149984.104234524,
|
||
|
|
"logits/rejected": -35314502.93975904,
|
||
|
|
"logps/chosen": -450.90482491856676,
|
||
|
|
"logps/rejected": -354.9996234939759,
|
||
|
|
"loss": 0.5681,
|
||
|
|
"loss/base_kto": 3.2480804920196533,
|
||
|
|
"metrics/advantage_var": 559.4739379882812,
|
||
|
|
"regularization/lipschitz_norm": 1339.9051513671875,
|
||
|
|
"regularization/w1": 1.2970281839370728,
|
||
|
|
"rewards/chosen": 0.4743180166238294,
|
||
|
|
"rewards/margins": 0.7994848337948572,
|
||
|
|
"rewards/rejected": -0.32516681717102786,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 16.029247283935547,
|
||
|
|
"kl": 13.296783447265625,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34697117.56899225,
|
||
|
|
"logits/rejected": -35621518.71496063,
|
||
|
|
"logps/chosen": -481.5494186046512,
|
||
|
|
"logps/rejected": -385.4054625984252,
|
||
|
|
"loss": 0.5587,
|
||
|
|
"loss/base_kto": 3.2124602794647217,
|
||
|
|
"metrics/advantage_var": 428.8362731933594,
|
||
|
|
"regularization/lipschitz_norm": 1298.7864990234375,
|
||
|
|
"regularization/w1": 1.2572253942489624,
|
||
|
|
"rewards/chosen": 0.5423459075218023,
|
||
|
|
"rewards/margins": 0.8369628710660322,
|
||
|
|
"rewards/rejected": -0.29461696354422984,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 13.329975128173828,
|
||
|
|
"kl": 13.94482707977295,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -34472352.1015873,
|
||
|
|
"logits/rejected": -36389752.51692308,
|
||
|
|
"logps/chosen": -486.62390873015875,
|
||
|
|
"logps/rejected": -357.45927884615384,
|
||
|
|
"loss": 0.5597,
|
||
|
|
"loss/base_kto": 3.1015193462371826,
|
||
|
|
"metrics/advantage_var": 529.8474731445312,
|
||
|
|
"regularization/lipschitz_norm": 1421.7562255859375,
|
||
|
|
"regularization/w1": 1.3762601613998413,
|
||
|
|
"rewards/chosen": 0.5987052796378968,
|
||
|
|
"rewards/margins": 0.9748003677163222,
|
||
|
|
"rewards/rejected": -0.37609508807842545,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 15.356056213378906,
|
||
|
|
"kl": 11.886622428894043,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -34654014.91190108,
|
||
|
|
"logits/rejected": -35670407.48183254,
|
||
|
|
"logps/chosen": -464.8797333848532,
|
||
|
|
"logps/rejected": -384.8746297393365,
|
||
|
|
"loss": 0.5671,
|
||
|
|
"loss/base_kto": 3.201629638671875,
|
||
|
|
"metrics/advantage_var": 484.70343017578125,
|
||
|
|
"regularization/lipschitz_norm": 1378.9920654296875,
|
||
|
|
"regularization/w1": 1.3348641395568848,
|
||
|
|
"rewards/chosen": 0.5062245954133259,
|
||
|
|
"rewards/margins": 0.8540332645653672,
|
||
|
|
"rewards/rejected": -0.3478086691520414,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 16.912874221801758,
|
||
|
|
"kl": 14.003283500671387,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -34599419.414925374,
|
||
|
|
"logits/rejected": -35416252.01311475,
|
||
|
|
"logps/chosen": -485.9853078358209,
|
||
|
|
"logps/rejected": -386.86342213114756,
|
||
|
|
"loss": 0.5719,
|
||
|
|
"loss/base_kto": 3.1810734272003174,
|
||
|
|
"metrics/advantage_var": 468.3973693847656,
|
||
|
|
"regularization/lipschitz_norm": 1440.0457763671875,
|
||
|
|
"regularization/w1": 1.3939642906188965,
|
||
|
|
"rewards/chosen": 0.5261796638147155,
|
||
|
|
"rewards/margins": 0.8673378209132042,
|
||
|
|
"rewards/rejected": -0.34115815709848873,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 13.959023475646973,
|
||
|
|
"kl": 18.680160522460938,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -32944703.795527156,
|
||
|
|
"logits/rejected": -36012818.00611621,
|
||
|
|
"logps/chosen": -466.327875399361,
|
||
|
|
"logps/rejected": -365.87370986238534,
|
||
|
|
"loss": 0.5582,
|
||
|
|
"loss/base_kto": 3.122089147567749,
|
||
|
|
"metrics/advantage_var": 494.7021484375,
|
||
|
|
"regularization/lipschitz_norm": 1387.5946044921875,
|
||
|
|
"regularization/w1": 1.3431915044784546,
|
||
|
|
"rewards/chosen": 0.6593199903591753,
|
||
|
|
"rewards/margins": 0.9139462265544043,
|
||
|
|
"rewards/rejected": -0.2546262361952289,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 17.987796783447266,
|
||
|
|
"kl": 15.049530029296875,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -34199591.44296296,
|
||
|
|
"logits/rejected": -34292866.32727273,
|
||
|
|
"logps/chosen": -487.01222222222225,
|
||
|
|
"logps/rejected": -378.01012396694216,
|
||
|
|
"loss": 0.5543,
|
||
|
|
"loss/base_kto": 3.1539852619171143,
|
||
|
|
"metrics/advantage_var": 451.4657287597656,
|
||
|
|
"regularization/lipschitz_norm": 1322.9229736328125,
|
||
|
|
"regularization/w1": 1.2805893421173096,
|
||
|
|
"rewards/chosen": 0.616356517650463,
|
||
|
|
"rewards/margins": 0.8641814970700136,
|
||
|
|
"rewards/rejected": -0.24782497941955062,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 19.532485961914062,
|
||
|
|
"kl": 15.837020874023438,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34549902.58227848,
|
||
|
|
"logits/rejected": -35127526.71604938,
|
||
|
|
"logps/chosen": -485.71073971518985,
|
||
|
|
"logps/rejected": -370.6300877700617,
|
||
|
|
"loss": 0.5568,
|
||
|
|
"loss/base_kto": 3.135880708694458,
|
||
|
|
"metrics/advantage_var": 483.2705078125,
|
||
|
|
"regularization/lipschitz_norm": 1361.748046875,
|
||
|
|
"regularization/w1": 1.3181720972061157,
|
||
|
|
"rewards/chosen": 0.5477567262287382,
|
||
|
|
"rewards/margins": 0.9015852270470615,
|
||
|
|
"rewards/rejected": -0.35382850081832323,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 19.168087005615234,
|
||
|
|
"kl": 13.672842979431152,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -33957715.434022255,
|
||
|
|
"logits/rejected": -36874866.039938554,
|
||
|
|
"logps/chosen": -496.45399443561206,
|
||
|
|
"logps/rejected": -351.21346966205834,
|
||
|
|
"loss": 0.5529,
|
||
|
|
"loss/base_kto": 3.1434247493743896,
|
||
|
|
"metrics/advantage_var": 430.8213806152344,
|
||
|
|
"regularization/lipschitz_norm": 1322.162109375,
|
||
|
|
"regularization/w1": 1.2798528671264648,
|
||
|
|
"rewards/chosen": 0.5340297680780505,
|
||
|
|
"rewards/margins": 0.883097256834831,
|
||
|
|
"rewards/rejected": -0.34906748875678045,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 20.155269622802734,
|
||
|
|
"kl": 13.380441665649414,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -34002827.95712098,
|
||
|
|
"logits/rejected": -34305286.94098884,
|
||
|
|
"logps/chosen": -480.2856527565084,
|
||
|
|
"logps/rejected": -387.265899122807,
|
||
|
|
"loss": 0.5584,
|
||
|
|
"loss/base_kto": 3.1513402462005615,
|
||
|
|
"metrics/advantage_var": 502.35137939453125,
|
||
|
|
"regularization/lipschitz_norm": 1359.7581787109375,
|
||
|
|
"regularization/w1": 1.3162459135055542,
|
||
|
|
"rewards/chosen": 0.5978419258620071,
|
||
|
|
"rewards/margins": 0.9165808967021936,
|
||
|
|
"rewards/rejected": -0.31873897084018643,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 14.516063690185547,
|
||
|
|
"kl": 14.525212287902832,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -33392608.0,
|
||
|
|
"logits/rejected": -35159033.6,
|
||
|
|
"logps/chosen": -503.3990234375,
|
||
|
|
"logps/rejected": -400.6938720703125,
|
||
|
|
"loss": 0.5642,
|
||
|
|
"loss/base_kto": 3.163792133331299,
|
||
|
|
"metrics/advantage_var": 471.1412048339844,
|
||
|
|
"regularization/lipschitz_norm": 1394.1875,
|
||
|
|
"regularization/w1": 1.3495734930038452,
|
||
|
|
"rewards/chosen": 0.5471538543701172,
|
||
|
|
"rewards/margins": 0.8902327299118042,
|
||
|
|
"rewards/rejected": -0.34307887554168703,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 15.617125511169434,
|
||
|
|
"kl": 12.250290870666504,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35133218.24203822,
|
||
|
|
"logits/rejected": -35582127.90184049,
|
||
|
|
"logps/chosen": -496.52164609872614,
|
||
|
|
"logps/rejected": -370.25340299079755,
|
||
|
|
"loss": 0.5481,
|
||
|
|
"loss/base_kto": 3.1153676509857178,
|
||
|
|
"metrics/advantage_var": 467.2294616699219,
|
||
|
|
"regularization/lipschitz_norm": 1310.98681640625,
|
||
|
|
"regularization/w1": 1.2690353393554688,
|
||
|
|
"rewards/chosen": 0.5348491790188346,
|
||
|
|
"rewards/margins": 0.9311423510945391,
|
||
|
|
"rewards/rejected": -0.3962931720757046,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 19.886005401611328,
|
||
|
|
"kl": 13.618847846984863,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -34087156.72955975,
|
||
|
|
"logits/rejected": -36125654.65838509,
|
||
|
|
"logps/chosen": -480.1741843553459,
|
||
|
|
"logps/rejected": -372.6374708850932,
|
||
|
|
"loss": 0.5528,
|
||
|
|
"loss/base_kto": 3.146395683288574,
|
||
|
|
"metrics/advantage_var": 431.99267578125,
|
||
|
|
"regularization/lipschitz_norm": 1317.941162109375,
|
||
|
|
"regularization/w1": 1.2757669687271118,
|
||
|
|
"rewards/chosen": 0.5684257843209513,
|
||
|
|
"rewards/margins": 0.8884076614287493,
|
||
|
|
"rewards/rejected": -0.31998187710779796,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 17.793807983398438,
|
||
|
|
"kl": 13.24231243133545,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -34019655.41272431,
|
||
|
|
"logits/rejected": -36141445.94902549,
|
||
|
|
"logps/chosen": -463.381015497553,
|
||
|
|
"logps/rejected": -380.266726011994,
|
||
|
|
"loss": 0.5516,
|
||
|
|
"loss/base_kto": 3.1404495239257812,
|
||
|
|
"metrics/advantage_var": 454.6252136230469,
|
||
|
|
"regularization/lipschitz_norm": 1314.20654296875,
|
||
|
|
"regularization/w1": 1.2721518278121948,
|
||
|
|
"rewards/chosen": 0.48604117015446574,
|
||
|
|
"rewards/margins": 0.9000160064667784,
|
||
|
|
"rewards/rejected": -0.4139748363123126,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 16.166582107543945,
|
||
|
|
"kl": 15.547632217407227,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -33855733.76,
|
||
|
|
"logits/rejected": -35213627.32698413,
|
||
|
|
"logps/chosen": -477.28346153846155,
|
||
|
|
"logps/rejected": -368.85835813492065,
|
||
|
|
"loss": 0.5536,
|
||
|
|
"loss/base_kto": 3.20768141746521,
|
||
|
|
"metrics/advantage_var": 407.5632629394531,
|
||
|
|
"regularization/lipschitz_norm": 1261.4091796875,
|
||
|
|
"regularization/w1": 1.2210441827774048,
|
||
|
|
"rewards/chosen": 0.486441650390625,
|
||
|
|
"rewards/margins": 0.8268258279467386,
|
||
|
|
"rewards/rejected": -0.3403841775561136,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 18.32387351989746,
|
||
|
|
"kl": 14.501152038574219,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -33962063.47462687,
|
||
|
|
"logits/rejected": -36594066.885245904,
|
||
|
|
"logps/chosen": -481.04869402985076,
|
||
|
|
"logps/rejected": -392.35432889344264,
|
||
|
|
"loss": 0.5656,
|
||
|
|
"loss/base_kto": 3.1842072010040283,
|
||
|
|
"metrics/advantage_var": 451.8145446777344,
|
||
|
|
"regularization/lipschitz_norm": 1384.595947265625,
|
||
|
|
"regularization/w1": 1.340288758277893,
|
||
|
|
"rewards/chosen": 0.5720315050722947,
|
||
|
|
"rewards/margins": 0.8596475719268029,
|
||
|
|
"rewards/rejected": -0.2876160668545082,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 21.38368034362793,
|
||
|
|
"kl": 19.178592681884766,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -33770083.48815166,
|
||
|
|
"logits/rejected": -34194788.10510046,
|
||
|
|
"logps/chosen": -498.29260466034754,
|
||
|
|
"logps/rejected": -371.72930351622875,
|
||
|
|
"loss": 0.5602,
|
||
|
|
"loss/base_kto": 3.1773126125335693,
|
||
|
|
"metrics/advantage_var": 452.7632751464844,
|
||
|
|
"regularization/lipschitz_norm": 1347.4056396484375,
|
||
|
|
"regularization/w1": 1.3042887449264526,
|
||
|
|
"rewards/chosen": 0.5842705074268119,
|
||
|
|
"rewards/margins": 0.8374482504806904,
|
||
|
|
"rewards/rejected": -0.25317774305387847,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 16.915109634399414,
|
||
|
|
"kl": 16.03028678894043,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -35029415.72413793,
|
||
|
|
"logits/rejected": -34908601.004893966,
|
||
|
|
"logps/chosen": -485.9177286356822,
|
||
|
|
"logps/rejected": -388.0621431484502,
|
||
|
|
"loss": 0.5582,
|
||
|
|
"loss/base_kto": 3.2086143493652344,
|
||
|
|
"metrics/advantage_var": 425.7274475097656,
|
||
|
|
"regularization/lipschitz_norm": 1298.6712646484375,
|
||
|
|
"regularization/w1": 1.2571138143539429,
|
||
|
|
"rewards/chosen": 0.56939697265625,
|
||
|
|
"rewards/margins": 0.8277747697192344,
|
||
|
|
"rewards/rejected": -0.2583777970629843,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 24.30204200744629,
|
||
|
|
"kl": 14.940142631530762,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -34003794.85990338,
|
||
|
|
"logits/rejected": -35394291.18057663,
|
||
|
|
"logps/chosen": -465.68599033816423,
|
||
|
|
"logps/rejected": -373.41426403641884,
|
||
|
|
"loss": 0.5592,
|
||
|
|
"loss/base_kto": 3.2054061889648438,
|
||
|
|
"metrics/advantage_var": 439.9033203125,
|
||
|
|
"regularization/lipschitz_norm": 1309.96923828125,
|
||
|
|
"regularization/w1": 1.268050193786621,
|
||
|
|
"rewards/chosen": 0.5584115290987319,
|
||
|
|
"rewards/margins": 0.8335664185426692,
|
||
|
|
"rewards/rejected": -0.2751548894439373,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 16.94352149963379,
|
||
|
|
"kl": 16.40333366394043,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -35490728.04907975,
|
||
|
|
"logits/rejected": -36010857.98726115,
|
||
|
|
"logps/chosen": -466.27779907975463,
|
||
|
|
"logps/rejected": -394.1244775079618,
|
||
|
|
"loss": 0.558,
|
||
|
|
"loss/base_kto": 3.1873714923858643,
|
||
|
|
"metrics/advantage_var": 437.2276916503906,
|
||
|
|
"regularization/lipschitz_norm": 1318.5452880859375,
|
||
|
|
"regularization/w1": 1.276351809501648,
|
||
|
|
"rewards/chosen": 0.5535883991264858,
|
||
|
|
"rewards/margins": 0.8391093536230274,
|
||
|
|
"rewards/rejected": -0.2855209544965416,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 17.528411865234375,
|
||
|
|
"kl": 16.821489334106445,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35454871.643312104,
|
||
|
|
"logits/rejected": -36273387.582822084,
|
||
|
|
"logps/chosen": -486.5666301751592,
|
||
|
|
"logps/rejected": -375.32254121932516,
|
||
|
|
"loss": 0.5601,
|
||
|
|
"loss/base_kto": 3.201885938644409,
|
||
|
|
"metrics/advantage_var": 461.4477233886719,
|
||
|
|
"regularization/lipschitz_norm": 1321.3861083984375,
|
||
|
|
"regularization/w1": 1.2791017293930054,
|
||
|
|
"rewards/chosen": 0.5660655999639231,
|
||
|
|
"rewards/margins": 0.8268592273693822,
|
||
|
|
"rewards/rejected": -0.26079362740545914,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 17.622358322143555,
|
||
|
|
"kl": 14.854166030883789,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -34862397.47112462,
|
||
|
|
"logits/rejected": -36349115.67845659,
|
||
|
|
"logps/chosen": -487.986797112462,
|
||
|
|
"logps/rejected": -382.080159766881,
|
||
|
|
"loss": 0.5516,
|
||
|
|
"loss/base_kto": 3.13665771484375,
|
||
|
|
"metrics/advantage_var": 469.099365234375,
|
||
|
|
"regularization/lipschitz_norm": 1318.103515625,
|
||
|
|
"regularization/w1": 1.2759240865707397,
|
||
|
|
"rewards/chosen": 0.5873298992864266,
|
||
|
|
"rewards/margins": 0.9101484714013282,
|
||
|
|
"rewards/rejected": -0.3228185721149015,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 18.09905433654785,
|
||
|
|
"kl": 12.205322265625,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -33807141.66966967,
|
||
|
|
"logits/rejected": -35700736.0,
|
||
|
|
"logps/chosen": -485.51783033033036,
|
||
|
|
"logps/rejected": -370.8291174674267,
|
||
|
|
"loss": 0.5639,
|
||
|
|
"loss/base_kto": 3.148710012435913,
|
||
|
|
"metrics/advantage_var": 479.14111328125,
|
||
|
|
"regularization/lipschitz_norm": 1407.3203125,
|
||
|
|
"regularization/w1": 1.3622859716415405,
|
||
|
|
"rewards/chosen": 0.5770113375093844,
|
||
|
|
"rewards/margins": 0.9141302256436729,
|
||
|
|
"rewards/rejected": -0.3371188881342885,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 21.58671760559082,
|
||
|
|
"kl": 13.516070365905762,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -34692467.650842264,
|
||
|
|
"logits/rejected": -35320722.57735247,
|
||
|
|
"logps/chosen": -488.7352603369066,
|
||
|
|
"logps/rejected": -369.4571371610845,
|
||
|
|
"loss": 0.5547,
|
||
|
|
"loss/base_kto": 3.1675968170166016,
|
||
|
|
"metrics/advantage_var": 478.0860290527344,
|
||
|
|
"regularization/lipschitz_norm": 1311.8924560546875,
|
||
|
|
"regularization/w1": 1.269911766052246,
|
||
|
|
"rewards/chosen": 0.5942296353817716,
|
||
|
|
"rewards/margins": 0.8910569364558347,
|
||
|
|
"rewards/rejected": -0.296827301074063,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 16.513416290283203,
|
||
|
|
"kl": 13.972925186157227,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -34216062.180094786,
|
||
|
|
"logits/rejected": -35726261.61360124,
|
||
|
|
"logps/chosen": -454.08101303317534,
|
||
|
|
"logps/rejected": -376.26019126738794,
|
||
|
|
"loss": 0.558,
|
||
|
|
"loss/base_kto": 3.2071805000305176,
|
||
|
|
"metrics/advantage_var": 427.6632995605469,
|
||
|
|
"regularization/lipschitz_norm": 1298.3660888671875,
|
||
|
|
"regularization/w1": 1.256818413734436,
|
||
|
|
"rewards/chosen": 0.5536692831753555,
|
||
|
|
"rewards/margins": 0.8289034289397609,
|
||
|
|
"rewards/rejected": -0.2752341457644054,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 16.91568374633789,
|
||
|
|
"kl": 14.527670860290527,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -34835875.566718996,
|
||
|
|
"logits/rejected": -36801327.377916016,
|
||
|
|
"logps/chosen": -482.87509811616957,
|
||
|
|
"logps/rejected": -383.3812694401244,
|
||
|
|
"loss": 0.555,
|
||
|
|
"loss/base_kto": 3.2087466716766357,
|
||
|
|
"metrics/advantage_var": 423.06982421875,
|
||
|
|
"regularization/lipschitz_norm": 1272.2606201171875,
|
||
|
|
"regularization/w1": 1.2315481901168823,
|
||
|
|
"rewards/chosen": 0.5398905468136774,
|
||
|
|
"rewards/margins": 0.8093527169770351,
|
||
|
|
"rewards/rejected": -0.2694621701633578,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 20.491783142089844,
|
||
|
|
"kl": 16.086137771606445,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -33287701.90808241,
|
||
|
|
"logits/rejected": -36201230.59476117,
|
||
|
|
"logps/chosen": -492.1697702060222,
|
||
|
|
"logps/rejected": -364.98613251155626,
|
||
|
|
"loss": 0.5539,
|
||
|
|
"loss/base_kto": 3.196310520172119,
|
||
|
|
"metrics/advantage_var": 497.85693359375,
|
||
|
|
"regularization/lipschitz_norm": 1275.878173828125,
|
||
|
|
"regularization/w1": 1.2350499629974365,
|
||
|
|
"rewards/chosen": 0.5651610747759014,
|
||
|
|
"rewards/margins": 0.8654061276366727,
|
||
|
|
"rewards/rejected": -0.30024505286077136,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 16.039255142211914,
|
||
|
|
"kl": 17.113508224487305,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -34850634.73291925,
|
||
|
|
"logits/rejected": -35910920.05031446,
|
||
|
|
"logps/chosen": -482.5238742236025,
|
||
|
|
"logps/rejected": -383.0166814072327,
|
||
|
|
"loss": 0.562,
|
||
|
|
"loss/base_kto": 3.234588384628296,
|
||
|
|
"metrics/advantage_var": 443.8955993652344,
|
||
|
|
"regularization/lipschitz_norm": 1303.3660888671875,
|
||
|
|
"regularization/w1": 1.2616583108901978,
|
||
|
|
"rewards/chosen": 0.5121984067170516,
|
||
|
|
"rewards/margins": 0.7845558675547457,
|
||
|
|
"rewards/rejected": -0.2723574608376941,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 16.25307846069336,
|
||
|
|
"kl": 14.860127449035645,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -34488639.80780781,
|
||
|
|
"logits/rejected": -36423980.19543974,
|
||
|
|
"logps/chosen": -488.32624812312315,
|
||
|
|
"logps/rejected": -366.43907776872965,
|
||
|
|
"loss": 0.5473,
|
||
|
|
"loss/base_kto": 3.14351749420166,
|
||
|
|
"metrics/advantage_var": 442.5341491699219,
|
||
|
|
"regularization/lipschitz_norm": 1275.9339599609375,
|
||
|
|
"regularization/w1": 1.2351040840148926,
|
||
|
|
"rewards/chosen": 0.5904500233876454,
|
||
|
|
"rewards/margins": 0.8832285604047718,
|
||
|
|
"rewards/rejected": -0.2927785370171264,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 15.692973136901855,
|
||
|
|
"kl": 16.274581909179688,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -33666748.283870965,
|
||
|
|
"logits/rejected": -35972406.303030305,
|
||
|
|
"logps/chosen": -486.7703629032258,
|
||
|
|
"logps/rejected": -382.0893465909091,
|
||
|
|
"loss": 0.5602,
|
||
|
|
"loss/base_kto": 3.148393392562866,
|
||
|
|
"metrics/advantage_var": 456.3197326660156,
|
||
|
|
"regularization/lipschitz_norm": 1376.9764404296875,
|
||
|
|
"regularization/w1": 1.3329132795333862,
|
||
|
|
"rewards/chosen": 0.583331544937626,
|
||
|
|
"rewards/margins": 0.9028989998825834,
|
||
|
|
"rewards/rejected": -0.31956745494495736,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 14.642328262329102,
|
||
|
|
"kl": 15.017115592956543,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34592242.87179487,
|
||
|
|
"logits/rejected": -36308703.222042136,
|
||
|
|
"logps/chosen": -491.4352375565611,
|
||
|
|
"logps/rejected": -387.99298521069693,
|
||
|
|
"loss": 0.5568,
|
||
|
|
"loss/base_kto": 3.1582484245300293,
|
||
|
|
"metrics/advantage_var": 435.5256042480469,
|
||
|
|
"regularization/lipschitz_norm": 1339.2635498046875,
|
||
|
|
"regularization/w1": 1.2964071035385132,
|
||
|
|
"rewards/chosen": 0.5732358354249152,
|
||
|
|
"rewards/margins": 0.8721980992772247,
|
||
|
|
"rewards/rejected": -0.29896226385230956,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 18.6578369140625,
|
||
|
|
"kl": 15.550450325012207,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -35819927.643312104,
|
||
|
|
"logits/rejected": -36710268.073619634,
|
||
|
|
"logps/chosen": -490.93197651273886,
|
||
|
|
"logps/rejected": -369.9531729294479,
|
||
|
|
"loss": 0.5568,
|
||
|
|
"loss/base_kto": 3.1798367500305176,
|
||
|
|
"metrics/advantage_var": 442.4677734375,
|
||
|
|
"regularization/lipschitz_norm": 1317.0311279296875,
|
||
|
|
"regularization/w1": 1.274886131286621,
|
||
|
|
"rewards/chosen": 0.5815582275390625,
|
||
|
|
"rewards/margins": 0.8676361832881998,
|
||
|
|
"rewards/rejected": -0.28607795574913725,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 14.409995079040527,
|
||
|
|
"kl": 14.426848411560059,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -35907821.54231975,
|
||
|
|
"logits/rejected": -36522507.16510903,
|
||
|
|
"logps/chosen": -483.8010384012539,
|
||
|
|
"logps/rejected": -366.04461156542055,
|
||
|
|
"loss": 0.5507,
|
||
|
|
"loss/base_kto": 3.1684281826019287,
|
||
|
|
"metrics/advantage_var": 424.572998046875,
|
||
|
|
"regularization/lipschitz_norm": 1278.3421630859375,
|
||
|
|
"regularization/w1": 1.237435221672058,
|
||
|
|
"rewards/chosen": 0.5785415792913646,
|
||
|
|
"rewards/margins": 0.8696636830686232,
|
||
|
|
"rewards/rejected": -0.29112210377725856,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 17.155927658081055,
|
||
|
|
"kl": 18.289228439331055,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -33932250.074074075,
|
||
|
|
"logits/rejected": -35081798.70106222,
|
||
|
|
"logps/chosen": -489.66928341384863,
|
||
|
|
"logps/rejected": -382.46156581942336,
|
||
|
|
"loss": 0.5602,
|
||
|
|
"loss/base_kto": 3.201462507247925,
|
||
|
|
"metrics/advantage_var": 452.5195007324219,
|
||
|
|
"regularization/lipschitz_norm": 1322.109375,
|
||
|
|
"regularization/w1": 1.279801845550537,
|
||
|
|
"rewards/chosen": 0.5439907694400412,
|
||
|
|
"rewards/margins": 0.8164065325888106,
|
||
|
|
"rewards/rejected": -0.2724157631487694,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 16.177122116088867,
|
||
|
|
"kl": 16.64619255065918,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -34925743.63754045,
|
||
|
|
"logits/rejected": -35050489.81268882,
|
||
|
|
"logps/chosen": -453.9710254854369,
|
||
|
|
"logps/rejected": -389.2960253021148,
|
||
|
|
"loss": 0.5608,
|
||
|
|
"loss/base_kto": 3.158618688583374,
|
||
|
|
"metrics/advantage_var": 503.90240478515625,
|
||
|
|
"regularization/lipschitz_norm": 1371.7763671875,
|
||
|
|
"regularization/w1": 1.327879548072815,
|
||
|
|
"rewards/chosen": 0.5825636286565787,
|
||
|
|
"rewards/margins": 0.9102988610991061,
|
||
|
|
"rewards/rejected": -0.3277352324425274,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 15.875944137573242,
|
||
|
|
"kl": 17.56245994567871,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -34364978.949429035,
|
||
|
|
"logits/rejected": -36102045.74512744,
|
||
|
|
"logps/chosen": -469.6697593800979,
|
||
|
|
"logps/rejected": -350.4022441904048,
|
||
|
|
"loss": 0.5606,
|
||
|
|
"loss/base_kto": 3.177584409713745,
|
||
|
|
"metrics/advantage_var": 445.9990234375,
|
||
|
|
"regularization/lipschitz_norm": 1350.1268310546875,
|
||
|
|
"regularization/w1": 1.3069227933883667,
|
||
|
|
"rewards/chosen": 0.5573854928693414,
|
||
|
|
"rewards/margins": 0.8269274034326144,
|
||
|
|
"rewards/rejected": -0.26954191056327303,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 14.446081161499023,
|
||
|
|
"kl": 14.45959186553955,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -35607916.36809816,
|
||
|
|
"logits/rejected": -34189116.33121019,
|
||
|
|
"logps/chosen": -443.21386119631904,
|
||
|
|
"logps/rejected": -357.3136942675159,
|
||
|
|
"loss": 0.5593,
|
||
|
|
"loss/base_kto": 3.21616792678833,
|
||
|
|
"metrics/advantage_var": 415.7434997558594,
|
||
|
|
"regularization/lipschitz_norm": 1300.151611328125,
|
||
|
|
"regularization/w1": 1.2585468292236328,
|
||
|
|
"rewards/chosen": 0.508663294505488,
|
||
|
|
"rewards/margins": 0.8264906042779552,
|
||
|
|
"rewards/rejected": -0.31782730977246715,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 16.99323081970215,
|
||
|
|
"kl": 14.399162292480469,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -34767189.87086614,
|
||
|
|
"logits/rejected": -36187744.0496124,
|
||
|
|
"logps/chosen": -468.33233267716537,
|
||
|
|
"logps/rejected": -375.66630329457365,
|
||
|
|
"loss": 0.5592,
|
||
|
|
"loss/base_kto": 3.196188449859619,
|
||
|
|
"metrics/advantage_var": 417.1357421875,
|
||
|
|
"regularization/lipschitz_norm": 1319.377197265625,
|
||
|
|
"regularization/w1": 1.2771570682525635,
|
||
|
|
"rewards/chosen": 0.5302483506090059,
|
||
|
|
"rewards/margins": 0.8518294828915882,
|
||
|
|
"rewards/rejected": -0.32158113228258234,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.97585917447635e+17,
|
||
|
|
"train_loss": 0.5860847269726952,
|
||
|
|
"train_runtime": 11055.2237,
|
||
|
|
"train_samples_per_second": 13.407,
|
||
|
|
"train_steps_per_second": 0.209
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.97585917447635e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|