191e0834a8
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed3 Source: Original Platform
2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 14.053450584411621,
|
|
"kl": 14.084790229797363,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37708246.03278688,
|
|
"logits/rejected": -37149159.61904762,
|
|
"logps/chosen": -464.8416076751118,
|
|
"logps/rejected": -395.8848265599343,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.8808434009552,
|
|
"metrics/advantage_var": 205.371826171875,
|
|
"regularization/mmd": 0.848584771156311,
|
|
"regularization/rkhs_norm": 163.50381469726562,
|
|
"rewards/chosen": 0.19078293642592678,
|
|
"rewards/margins": 0.08929381140748595,
|
|
"rewards/rejected": 0.10148912501844083,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 19.398693084716797,
|
|
"kl": 5.930083751678467,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36159434.281967215,
|
|
"logits/rejected": -37062790.49552239,
|
|
"logps/chosen": -484.65245901639344,
|
|
"logps/rejected": -372.5240904850746,
|
|
"loss": 0.5865,
|
|
"loss/base_kto": 3.9244744777679443,
|
|
"metrics/advantage_var": 166.6823272705078,
|
|
"regularization/mmd": 0.7675223350524902,
|
|
"regularization/rkhs_norm": 147.8848419189453,
|
|
"rewards/chosen": -0.008687799485003362,
|
|
"rewards/margins": 0.05910607796004678,
|
|
"rewards/rejected": -0.06779387744505014,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 18.025745391845703,
|
|
"kl": 3.7812564373016357,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35643044.16507936,
|
|
"logits/rejected": -37023447.82769231,
|
|
"logps/chosen": -482.2001488095238,
|
|
"logps/rejected": -347.13278846153844,
|
|
"loss": 0.5804,
|
|
"loss/base_kto": 3.877356767654419,
|
|
"metrics/advantage_var": 169.86289978027344,
|
|
"regularization/mmd": 0.7659773230552673,
|
|
"regularization/rkhs_norm": 147.587158203125,
|
|
"rewards/chosen": -0.056766740102616564,
|
|
"rewards/margins": 0.11799921354822479,
|
|
"rewards/rejected": -0.17476595365084135,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 19.38488006591797,
|
|
"kl": 7.673327922821045,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35561628.038095236,
|
|
"logits/rejected": -37948948.48,
|
|
"logps/chosen": -490.85218253968253,
|
|
"logps/rejected": -381.0875721153846,
|
|
"loss": 0.5867,
|
|
"loss/base_kto": 3.9031662940979004,
|
|
"metrics/advantage_var": 182.56967163085938,
|
|
"regularization/mmd": 0.7907209396362305,
|
|
"regularization/rkhs_norm": 152.35472106933594,
|
|
"rewards/chosen": -0.001493631847321041,
|
|
"rewards/margins": 0.059328963602273306,
|
|
"rewards/rejected": -0.06082259544959435,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 15.992303848266602,
|
|
"kl": 5.130296230316162,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35266242.93206951,
|
|
"logits/rejected": -35483097.22411128,
|
|
"logps/chosen": -497.24777843601896,
|
|
"logps/rejected": -363.547478748068,
|
|
"loss": 0.5756,
|
|
"loss/base_kto": 3.837254047393799,
|
|
"metrics/advantage_var": 164.3052215576172,
|
|
"regularization/mmd": 0.7674898505210876,
|
|
"regularization/rkhs_norm": 147.8785858154297,
|
|
"rewards/chosen": 0.06861413410301269,
|
|
"rewards/margins": 0.15924977870029763,
|
|
"rewards/rejected": -0.09063564459728495,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 17.491933822631836,
|
|
"kl": 10.688410758972168,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35228113.592867754,
|
|
"logits/rejected": -37261440.21087315,
|
|
"logps/chosen": -483.4838410104012,
|
|
"logps/rejected": -377.4160317133443,
|
|
"loss": 0.5781,
|
|
"loss/base_kto": 3.83256459236145,
|
|
"metrics/advantage_var": 180.60057067871094,
|
|
"regularization/mmd": 0.7920131087303162,
|
|
"regularization/rkhs_norm": 152.6036834716797,
|
|
"rewards/chosen": 0.21083334119323457,
|
|
"rewards/margins": 0.1491044505087077,
|
|
"rewards/rejected": 0.061728890684526874,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 14.425165176391602,
|
|
"kl": 7.135074138641357,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35280412.981132075,
|
|
"logits/rejected": -36528099.37888199,
|
|
"logps/chosen": -489.5138561320755,
|
|
"logps/rejected": -407.1116556677019,
|
|
"loss": 0.5805,
|
|
"loss/base_kto": 3.820751667022705,
|
|
"metrics/advantage_var": 190.59127807617188,
|
|
"regularization/mmd": 0.8236228823661804,
|
|
"regularization/rkhs_norm": 158.6942138671875,
|
|
"rewards/chosen": 0.10541529625466785,
|
|
"rewards/margins": 0.16605717735628797,
|
|
"rewards/rejected": -0.060641881101620124,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 15.305477142333984,
|
|
"kl": 8.461416244506836,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36388022.18404908,
|
|
"logits/rejected": -37817011.36305732,
|
|
"logps/chosen": -460.36205904907973,
|
|
"logps/rejected": -385.54371516719743,
|
|
"loss": 0.5849,
|
|
"loss/base_kto": 3.8157455921173096,
|
|
"metrics/advantage_var": 212.3812255859375,
|
|
"regularization/mmd": 0.8636655211448669,
|
|
"regularization/rkhs_norm": 166.4095458984375,
|
|
"rewards/chosen": 0.13132327027116086,
|
|
"rewards/margins": 0.17102924531166955,
|
|
"rewards/rejected": -0.039705975040508684,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 19.524709701538086,
|
|
"kl": 6.943305492401123,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36245105.59485531,
|
|
"logits/rejected": -37543674.55319149,
|
|
"logps/chosen": -499.3322950160772,
|
|
"logps/rejected": -372.20426481762917,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.801764726638794,
|
|
"metrics/advantage_var": 225.54039001464844,
|
|
"regularization/mmd": 0.8828476071357727,
|
|
"regularization/rkhs_norm": 170.10552978515625,
|
|
"rewards/chosen": 0.0631781636327026,
|
|
"rewards/margins": 0.18249577203410838,
|
|
"rewards/rejected": -0.11931760840140578,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 15.128318786621094,
|
|
"kl": 9.885144233703613,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37237989.3006135,
|
|
"logits/rejected": -37037284.280254774,
|
|
"logps/chosen": -476.03561157975463,
|
|
"logps/rejected": -384.0027617436306,
|
|
"loss": 0.5767,
|
|
"loss/base_kto": 3.7704529762268066,
|
|
"metrics/advantage_var": 202.85032653808594,
|
|
"regularization/mmd": 0.8433179259300232,
|
|
"regularization/rkhs_norm": 162.48899841308594,
|
|
"rewards/chosen": 0.16586137549277463,
|
|
"rewards/margins": 0.20762090898008242,
|
|
"rewards/rejected": -0.0417595334873078,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.586498260498047,
|
|
"kl": 12.858744621276855,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37259064.1152,
|
|
"logits/rejected": -36553679.53587786,
|
|
"logps/chosen": -488.2318,
|
|
"logps/rejected": -355.60395992366415,
|
|
"loss": 0.5775,
|
|
"loss/base_kto": 3.756343126296997,
|
|
"metrics/advantage_var": 220.56626892089844,
|
|
"regularization/mmd": 0.8633705377578735,
|
|
"regularization/rkhs_norm": 166.35272216796875,
|
|
"rewards/chosen": 0.224130712890625,
|
|
"rewards/margins": 0.23478865964467288,
|
|
"rewards/rejected": -0.010657946754047888,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 19.05987548828125,
|
|
"kl": 13.861737251281738,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36267047.56723338,
|
|
"logits/rejected": -37761674.312796205,
|
|
"logps/chosen": -465.7286514683153,
|
|
"logps/rejected": -363.83925750394945,
|
|
"loss": 0.5814,
|
|
"loss/base_kto": 3.753122091293335,
|
|
"metrics/advantage_var": 251.46963500976562,
|
|
"regularization/mmd": 0.8980002403259277,
|
|
"regularization/rkhs_norm": 173.0251007080078,
|
|
"rewards/chosen": 0.25279615540777145,
|
|
"rewards/margins": 0.2277779093431409,
|
|
"rewards/rejected": 0.02501824606463054,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 14.120685577392578,
|
|
"kl": 9.18895435333252,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37800140.475435816,
|
|
"logits/rejected": -39671281.010785826,
|
|
"logps/chosen": -495.6339144215531,
|
|
"logps/rejected": -386.2925895608629,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.7914979457855225,
|
|
"metrics/advantage_var": 271.4533386230469,
|
|
"regularization/mmd": 0.9776763916015625,
|
|
"regularization/rkhs_norm": 188.376953125,
|
|
"rewards/chosen": 0.06896091225407959,
|
|
"rewards/margins": 0.19787586140963143,
|
|
"rewards/rejected": -0.12891494915555182,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 13.508169174194336,
|
|
"kl": 7.100071907043457,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36699683.87261146,
|
|
"logits/rejected": -37723638.57668712,
|
|
"logps/chosen": -483.71695859872614,
|
|
"logps/rejected": -370.41739359662574,
|
|
"loss": 0.5752,
|
|
"loss/base_kto": 3.7420413494110107,
|
|
"metrics/advantage_var": 209.5242156982422,
|
|
"regularization/mmd": 0.8596655130386353,
|
|
"regularization/rkhs_norm": 165.63880920410156,
|
|
"rewards/chosen": 0.11126981115644904,
|
|
"rewards/margins": 0.23646974341865906,
|
|
"rewards/rejected": -0.12519993226221002,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 17.211875915527344,
|
|
"kl": 9.971189498901367,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37893339.54303599,
|
|
"logits/rejected": -38475239.23868955,
|
|
"logps/chosen": -485.0159428794992,
|
|
"logps/rejected": -376.35547484399376,
|
|
"loss": 0.5773,
|
|
"loss/base_kto": 3.718311309814453,
|
|
"metrics/advantage_var": 242.1728057861328,
|
|
"regularization/mmd": 0.9001728296279907,
|
|
"regularization/rkhs_norm": 173.44369506835938,
|
|
"rewards/chosen": 0.17436685099474888,
|
|
"rewards/margins": 0.2696254001062446,
|
|
"rewards/rejected": -0.09525854911149571,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 14.395646095275879,
|
|
"kl": 8.684496879577637,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35847447.831831835,
|
|
"logits/rejected": -37449934.80130293,
|
|
"logps/chosen": -484.40099474474476,
|
|
"logps/rejected": -347.8960199511401,
|
|
"loss": 0.572,
|
|
"loss/base_kto": 3.746997117996216,
|
|
"metrics/advantage_var": 195.2505340576172,
|
|
"regularization/mmd": 0.8292547464370728,
|
|
"regularization/rkhs_norm": 159.77932739257812,
|
|
"rewards/chosen": 0.20511716550534909,
|
|
"rewards/margins": 0.23176708171620705,
|
|
"rewards/rejected": -0.026649916210857976,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 13.376764297485352,
|
|
"kl": 10.373708724975586,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36108411.37349398,
|
|
"logits/rejected": -37652566.44155844,
|
|
"logps/chosen": -477.92131024096386,
|
|
"logps/rejected": -387.2690239448052,
|
|
"loss": 0.5827,
|
|
"loss/base_kto": 3.7676455974578857,
|
|
"metrics/advantage_var": 232.1603240966797,
|
|
"regularization/mmd": 0.8936416506767273,
|
|
"regularization/rkhs_norm": 172.185302734375,
|
|
"rewards/chosen": 0.17485512882830148,
|
|
"rewards/margins": 0.20034630608995127,
|
|
"rewards/rejected": -0.02549117726164979,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 16.88874626159668,
|
|
"kl": 12.72825813293457,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35535576.39553429,
|
|
"logits/rejected": -36849526.003062785,
|
|
"logps/chosen": -499.45858253588517,
|
|
"logps/rejected": -354.64720520673814,
|
|
"loss": 0.5763,
|
|
"loss/base_kto": 3.7054126262664795,
|
|
"metrics/advantage_var": 231.28042602539062,
|
|
"regularization/mmd": 0.9050853848457336,
|
|
"regularization/rkhs_norm": 174.3902587890625,
|
|
"rewards/chosen": 0.23496967905826355,
|
|
"rewards/margins": 0.2719173996896045,
|
|
"rewards/rejected": -0.03694772063134093,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 17.518848419189453,
|
|
"kl": 4.519381046295166,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36748237.2096,
|
|
"logits/rejected": -38333388.40916031,
|
|
"logps/chosen": -496.4691,
|
|
"logps/rejected": -376.2216603053435,
|
|
"loss": 0.5824,
|
|
"loss/base_kto": 3.7478439807891846,
|
|
"metrics/advantage_var": 243.8812255859375,
|
|
"regularization/mmd": 0.9114912152290344,
|
|
"regularization/rkhs_norm": 175.62451171875,
|
|
"rewards/chosen": 0.004214739990234375,
|
|
"rewards/margins": 0.2471240622484047,
|
|
"rewards/rejected": -0.24290932225817033,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 14.806842803955078,
|
|
"kl": 11.1155424118042,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36462232.81230769,
|
|
"logits/rejected": -38180890.006349206,
|
|
"logps/chosen": -489.86875,
|
|
"logps/rejected": -383.138244047619,
|
|
"loss": 0.5894,
|
|
"loss/base_kto": 3.776733160018921,
|
|
"metrics/advantage_var": 258.5162353515625,
|
|
"regularization/mmd": 0.9383435249328613,
|
|
"regularization/rkhs_norm": 180.7983856201172,
|
|
"rewards/chosen": 0.20011434701772837,
|
|
"rewards/margins": 0.1867742795327062,
|
|
"rewards/rejected": 0.013340067485022167,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 15.332754135131836,
|
|
"kl": 13.656012535095215,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36453072.59259259,
|
|
"logits/rejected": -39898773.06329114,
|
|
"logps/chosen": -492.7653838734568,
|
|
"logps/rejected": -356.0156744462025,
|
|
"loss": 0.5763,
|
|
"loss/base_kto": 3.6899516582489014,
|
|
"metrics/advantage_var": 265.62066650390625,
|
|
"regularization/mmd": 0.9202811121940613,
|
|
"regularization/rkhs_norm": 177.31814575195312,
|
|
"rewards/chosen": 0.2067541899504485,
|
|
"rewards/margins": 0.299070467369913,
|
|
"rewards/rejected": -0.09231627741946449,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 16.962617874145508,
|
|
"kl": 9.340049743652344,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35479123.96141479,
|
|
"logits/rejected": -37338006.17629179,
|
|
"logps/chosen": -478.0871684083601,
|
|
"logps/rejected": -374.5095459726444,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.7416157722473145,
|
|
"metrics/advantage_var": 263.5098571777344,
|
|
"regularization/mmd": 0.9615511894226074,
|
|
"regularization/rkhs_norm": 185.26998901367188,
|
|
"rewards/chosen": 0.10677789031884295,
|
|
"rewards/margins": 0.24046990413764757,
|
|
"rewards/rejected": -0.1336920138188046,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 15.43411922454834,
|
|
"kl": 13.604727745056152,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36095730.03636364,
|
|
"logits/rejected": -37040778.73548387,
|
|
"logps/chosen": -459.4022727272727,
|
|
"logps/rejected": -381.1258064516129,
|
|
"loss": 0.5752,
|
|
"loss/base_kto": 3.712679624557495,
|
|
"metrics/advantage_var": 219.68603515625,
|
|
"regularization/mmd": 0.8886733055114746,
|
|
"regularization/rkhs_norm": 171.22801208496094,
|
|
"rewards/chosen": 0.27248995231859613,
|
|
"rewards/margins": 0.27280020872285754,
|
|
"rewards/rejected": -0.0003102564042614352,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 16.855188369750977,
|
|
"kl": 6.829953193664551,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37790243.68233387,
|
|
"logits/rejected": -37987749.64705882,
|
|
"logps/chosen": -501.64049837925444,
|
|
"logps/rejected": -366.2626319758673,
|
|
"loss": 0.5844,
|
|
"loss/base_kto": 3.7384986877441406,
|
|
"metrics/advantage_var": 248.6357421875,
|
|
"regularization/mmd": 0.9368970990180969,
|
|
"regularization/rkhs_norm": 180.51968383789062,
|
|
"rewards/chosen": 0.0473186672216876,
|
|
"rewards/margins": 0.2532673717297829,
|
|
"rewards/rejected": -0.2059487045080953,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 12.89775276184082,
|
|
"kl": 10.080784797668457,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35414956.88820827,
|
|
"logits/rejected": -36153588.97607656,
|
|
"logps/chosen": -495.36557235834607,
|
|
"logps/rejected": -381.60827850877195,
|
|
"loss": 0.5779,
|
|
"loss/base_kto": 3.7460522651672363,
|
|
"metrics/advantage_var": 224.7728271484375,
|
|
"regularization/mmd": 0.8774681091308594,
|
|
"regularization/rkhs_norm": 169.0690155029297,
|
|
"rewards/chosen": 0.15558583374958126,
|
|
"rewards/margins": 0.23520338424814213,
|
|
"rewards/rejected": -0.07961755049856085,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 20.087554931640625,
|
|
"kl": 12.734642028808594,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36687219.512195125,
|
|
"logits/rejected": -37605586.05128205,
|
|
"logps/chosen": -477.8283631859756,
|
|
"logps/rejected": -365.99173677884613,
|
|
"loss": 0.5833,
|
|
"loss/base_kto": 3.7508270740509033,
|
|
"metrics/advantage_var": 243.4734649658203,
|
|
"regularization/mmd": 0.9156200289726257,
|
|
"regularization/rkhs_norm": 176.42002868652344,
|
|
"rewards/chosen": 0.20695081571253335,
|
|
"rewards/margins": 0.18733089081267404,
|
|
"rewards/rejected": 0.019619924899859306,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 19.172365188598633,
|
|
"kl": 17.163976669311523,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37937309.786163524,
|
|
"logits/rejected": -39212483.57763975,
|
|
"logps/chosen": -482.9131289308176,
|
|
"logps/rejected": -378.0909355590062,
|
|
"loss": 0.5822,
|
|
"loss/base_kto": 3.7559926509857178,
|
|
"metrics/advantage_var": 228.3892059326172,
|
|
"regularization/mmd": 0.901911199092865,
|
|
"regularization/rkhs_norm": 173.7786407470703,
|
|
"rewards/chosen": 0.2554650216732385,
|
|
"rewards/margins": 0.1860051302058775,
|
|
"rewards/rejected": 0.06945989146736098,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 14.490078926086426,
|
|
"kl": 8.486918449401855,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35820647.04402516,
|
|
"logits/rejected": -38229748.86956522,
|
|
"logps/chosen": -474.60131682389937,
|
|
"logps/rejected": -359.96261160714283,
|
|
"loss": 0.5822,
|
|
"loss/base_kto": 3.7075035572052,
|
|
"metrics/advantage_var": 282.5897216796875,
|
|
"regularization/mmd": 0.9503172039985657,
|
|
"regularization/rkhs_norm": 183.10545349121094,
|
|
"rewards/chosen": 0.16441621120620822,
|
|
"rewards/margins": 0.254419959323975,
|
|
"rewards/rejected": -0.09000374811776676,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 16.418272018432617,
|
|
"kl": 15.72160816192627,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36241930.82779456,
|
|
"logits/rejected": -36921887.482200645,
|
|
"logps/chosen": -483.28455438066464,
|
|
"logps/rejected": -365.03580097087377,
|
|
"loss": 0.5805,
|
|
"loss/base_kto": 3.7439968585968018,
|
|
"metrics/advantage_var": 237.0535430908203,
|
|
"regularization/mmd": 0.9000579714775085,
|
|
"regularization/rkhs_norm": 173.42156982421875,
|
|
"rewards/chosen": 0.19201597922518174,
|
|
"rewards/margins": 0.21814218032397256,
|
|
"rewards/rejected": -0.02612620109879083,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 14.830126762390137,
|
|
"kl": 9.746199607849121,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36975607.83413078,
|
|
"logits/rejected": -37381739.41807044,
|
|
"logps/chosen": -482.85566188197765,
|
|
"logps/rejected": -373.07132944104137,
|
|
"loss": 0.5802,
|
|
"loss/base_kto": 3.7088935375213623,
|
|
"metrics/advantage_var": 251.599609375,
|
|
"regularization/mmd": 0.9325283169746399,
|
|
"regularization/rkhs_norm": 179.6779022216797,
|
|
"rewards/chosen": 0.11735965418473385,
|
|
"rewards/margins": 0.2652162453086332,
|
|
"rewards/rejected": -0.14785659112389932,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 14.273622512817383,
|
|
"kl": 13.685553550720215,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37599927.85829308,
|
|
"logits/rejected": -37447173.438543245,
|
|
"logps/chosen": -505.41450281803543,
|
|
"logps/rejected": -389.41891122913506,
|
|
"loss": 0.5825,
|
|
"loss/base_kto": 3.7360076904296875,
|
|
"metrics/advantage_var": 243.45420837402344,
|
|
"regularization/mmd": 0.9236852526664734,
|
|
"regularization/rkhs_norm": 177.9740447998047,
|
|
"rewards/chosen": 0.14979403337610708,
|
|
"rewards/margins": 0.2217305177288788,
|
|
"rewards/rejected": -0.07193648435277172,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 18.475658416748047,
|
|
"kl": 7.479175567626953,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35724883.96141479,
|
|
"logits/rejected": -36531389.86018237,
|
|
"logps/chosen": -480.787027733119,
|
|
"logps/rejected": -383.4475683890578,
|
|
"loss": 0.5852,
|
|
"loss/base_kto": 3.80198073387146,
|
|
"metrics/advantage_var": 219.8570098876953,
|
|
"regularization/mmd": 0.8793790936470032,
|
|
"regularization/rkhs_norm": 169.4372100830078,
|
|
"rewards/chosen": 0.018721825823523223,
|
|
"rewards/margins": 0.156587802545568,
|
|
"rewards/rejected": -0.1378659767220448,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 24.893247604370117,
|
|
"kl": 11.908342361450195,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36665331.512195125,
|
|
"logits/rejected": -37058852.102564104,
|
|
"logps/chosen": -504.6525342987805,
|
|
"logps/rejected": -382.8387419871795,
|
|
"loss": 0.5837,
|
|
"loss/base_kto": 3.6899306774139404,
|
|
"metrics/advantage_var": 274.0852966308594,
|
|
"regularization/mmd": 0.979768693447113,
|
|
"regularization/rkhs_norm": 188.7800750732422,
|
|
"rewards/chosen": 0.21740799415402295,
|
|
"rewards/margins": 0.29339041808309074,
|
|
"rewards/rejected": -0.07598242392906776,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 15.199433326721191,
|
|
"kl": 11.275158882141113,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35797112.662420385,
|
|
"logits/rejected": -35241280.392638035,
|
|
"logps/chosen": -470.94740246815286,
|
|
"logps/rejected": -366.4884969325153,
|
|
"loss": 0.5754,
|
|
"loss/base_kto": 3.6965701580047607,
|
|
"metrics/advantage_var": 246.820068359375,
|
|
"regularization/mmd": 0.9064862132072449,
|
|
"regularization/rkhs_norm": 174.66017150878906,
|
|
"rewards/chosen": 0.12277058886874254,
|
|
"rewards/margins": 0.2659353815439247,
|
|
"rewards/rejected": -0.14316479267518215,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 30.429025650024414,
|
|
"kl": 12.471336364746094,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37540932.060422964,
|
|
"logits/rejected": -37486260.60841424,
|
|
"logps/chosen": -476.2478285498489,
|
|
"logps/rejected": -370.8467334142395,
|
|
"loss": 0.5843,
|
|
"loss/base_kto": 3.714616060256958,
|
|
"metrics/advantage_var": 258.0834045410156,
|
|
"regularization/mmd": 0.9598900079727173,
|
|
"regularization/rkhs_norm": 184.9499053955078,
|
|
"rewards/chosen": 0.23435517740393694,
|
|
"rewards/margins": 0.2522826512067195,
|
|
"rewards/rejected": -0.017927473802782572,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 21.522136688232422,
|
|
"kl": 14.137832641601562,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35931124.78247261,
|
|
"logits/rejected": -35518237.15444618,
|
|
"logps/chosen": -493.339446400626,
|
|
"logps/rejected": -374.41558599843995,
|
|
"loss": 0.5708,
|
|
"loss/base_kto": 3.6544816493988037,
|
|
"metrics/advantage_var": 237.94754028320312,
|
|
"regularization/mmd": 0.9120274782180786,
|
|
"regularization/rkhs_norm": 175.7278289794922,
|
|
"rewards/chosen": 0.23498907671288147,
|
|
"rewards/margins": 0.31253378448874136,
|
|
"rewards/rejected": -0.07754470777585987,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 46.376590728759766,
|
|
"kl": 8.488295555114746,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36338688.0,
|
|
"logits/rejected": -37239142.08588957,
|
|
"logps/chosen": -476.63883359872614,
|
|
"logps/rejected": -360.99616564417175,
|
|
"loss": 0.5908,
|
|
"loss/base_kto": 3.7207353115081787,
|
|
"metrics/advantage_var": 326.5664978027344,
|
|
"regularization/mmd": 1.005431056022644,
|
|
"regularization/rkhs_norm": 193.7246551513672,
|
|
"rewards/chosen": 0.106472063975729,
|
|
"rewards/margins": 0.26960543945951687,
|
|
"rewards/rejected": -0.16313337548378787,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 18.825122833251953,
|
|
"kl": 11.927857398986816,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35359502.77691108,
|
|
"logits/rejected": -36505763.45539906,
|
|
"logps/chosen": -507.9542219188768,
|
|
"logps/rejected": -371.8377591940532,
|
|
"loss": 0.5804,
|
|
"loss/base_kto": 3.7018089294433594,
|
|
"metrics/advantage_var": 255.86717224121094,
|
|
"regularization/mmd": 0.9415954947471619,
|
|
"regularization/rkhs_norm": 181.4249267578125,
|
|
"rewards/chosen": 0.18441221382986178,
|
|
"rewards/margins": 0.26960073772087495,
|
|
"rewards/rejected": -0.08518852389101318,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 15.647745132446289,
|
|
"kl": 13.50858211517334,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36221784.26209048,
|
|
"logits/rejected": -37133683.34065934,
|
|
"logps/chosen": -488.05845358814355,
|
|
"logps/rejected": -362.4831976059655,
|
|
"loss": 0.58,
|
|
"loss/base_kto": 3.516514539718628,
|
|
"metrics/advantage_var": 402.0416564941406,
|
|
"regularization/mmd": 1.1236242055892944,
|
|
"regularization/rkhs_norm": 216.49790954589844,
|
|
"rewards/chosen": 0.3366479977802628,
|
|
"rewards/margins": 0.4960871086484682,
|
|
"rewards/rejected": -0.15943911086820545,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 17.787534713745117,
|
|
"kl": 15.213953971862793,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -34774170.38769231,
|
|
"logits/rejected": -35415644.647619046,
|
|
"logps/chosen": -485.1367307692308,
|
|
"logps/rejected": -384.04761904761904,
|
|
"loss": 0.5694,
|
|
"loss/base_kto": 3.1502323150634766,
|
|
"metrics/advantage_var": 653.2338256835938,
|
|
"regularization/mmd": 1.4050582647323608,
|
|
"regularization/rkhs_norm": 270.72418212890625,
|
|
"rewards/chosen": 0.5699268517127404,
|
|
"rewards/margins": 0.9771774407504388,
|
|
"rewards/rejected": -0.4072505890376984,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 11.427125930786133,
|
|
"kl": 20.39912986755371,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34912628.938271604,
|
|
"logits/rejected": -36087665.41772152,
|
|
"logps/chosen": -494.91415895061726,
|
|
"logps/rejected": -398.68448378164555,
|
|
"loss": 0.5721,
|
|
"loss/base_kto": 3.2060418128967285,
|
|
"metrics/advantage_var": 587.7279663085938,
|
|
"regularization/mmd": 1.37111496925354,
|
|
"regularization/rkhs_norm": 264.1839904785156,
|
|
"rewards/chosen": 0.6314134009090471,
|
|
"rewards/margins": 0.843188902981152,
|
|
"rewards/rejected": -0.21177550207210494,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 15.561081886291504,
|
|
"kl": 18.192419052124023,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35077527.29146538,
|
|
"logits/rejected": -35928986.99848255,
|
|
"logps/chosen": -490.0734702093398,
|
|
"logps/rejected": -395.0282150986343,
|
|
"loss": 0.5676,
|
|
"loss/base_kto": 3.1527559757232666,
|
|
"metrics/advantage_var": 599.448974609375,
|
|
"regularization/mmd": 1.388403058052063,
|
|
"regularization/rkhs_norm": 267.5150451660156,
|
|
"rewards/chosen": 0.5970317453577898,
|
|
"rewards/margins": 0.9376627469937913,
|
|
"rewards/rejected": -0.3406310016360015,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 11.20211410522461,
|
|
"kl": 5.270300388336182,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35142483.968,
|
|
"logits/rejected": -36052635.55419847,
|
|
"logps/chosen": -487.6436,
|
|
"logps/rejected": -390.7299141221374,
|
|
"loss": 0.5675,
|
|
"loss/base_kto": 3.196239709854126,
|
|
"metrics/advantage_var": 593.168212890625,
|
|
"regularization/mmd": 1.3436870574951172,
|
|
"regularization/rkhs_norm": 258.8992614746094,
|
|
"rewards/chosen": 0.2894452392578125,
|
|
"rewards/margins": 0.9110468856229127,
|
|
"rewards/rejected": -0.6216016463651002,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 9.931262969970703,
|
|
"kl": 16.42917823791504,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35681132.338557996,
|
|
"logits/rejected": -36317927.27725857,
|
|
"logps/chosen": -469.24226097178683,
|
|
"logps/rejected": -362.39047897196264,
|
|
"loss": 0.5669,
|
|
"loss/base_kto": 3.1452138423919678,
|
|
"metrics/advantage_var": 632.6906127929688,
|
|
"regularization/mmd": 1.3901761770248413,
|
|
"regularization/rkhs_norm": 267.856689453125,
|
|
"rewards/chosen": 0.617636271031299,
|
|
"rewards/margins": 0.9493054931886199,
|
|
"rewards/rejected": -0.33166922215732086,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 15.128028869628906,
|
|
"kl": 9.139902114868164,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35025431.63076923,
|
|
"logits/rejected": -36179308.08888889,
|
|
"logps/chosen": -474.06264423076925,
|
|
"logps/rejected": -384.5904265873016,
|
|
"loss": 0.5681,
|
|
"loss/base_kto": 3.221485137939453,
|
|
"metrics/advantage_var": 543.8860473632812,
|
|
"regularization/mmd": 1.323137640953064,
|
|
"regularization/rkhs_norm": 254.93984985351562,
|
|
"rewards/chosen": 0.4105254187950721,
|
|
"rewards/margins": 0.8529315796643677,
|
|
"rewards/rejected": -0.44240616086929563,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 9.487354278564453,
|
|
"kl": 17.894189834594727,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35247873.1797235,
|
|
"logits/rejected": -36381306.91255962,
|
|
"logps/chosen": -487.01939324116745,
|
|
"logps/rejected": -355.4643034578696,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.074049472808838,
|
|
"metrics/advantage_var": 624.259765625,
|
|
"regularization/mmd": 1.4144279956817627,
|
|
"regularization/rkhs_norm": 272.52947998046875,
|
|
"rewards/chosen": 0.7293932749195948,
|
|
"rewards/margins": 1.0139652758325022,
|
|
"rewards/rejected": -0.2845720009129074,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 9.786842346191406,
|
|
"kl": 11.190815925598145,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -33631526.6645469,
|
|
"logits/rejected": -34471942.29185868,
|
|
"logps/chosen": -473.63533386327504,
|
|
"logps/rejected": -348.2914746543779,
|
|
"loss": 0.5653,
|
|
"loss/base_kto": 3.220233201980591,
|
|
"metrics/advantage_var": 540.1288452148438,
|
|
"regularization/mmd": 1.3020673990249634,
|
|
"regularization/rkhs_norm": 250.8800506591797,
|
|
"rewards/chosen": 0.47080467236250495,
|
|
"rewards/margins": 0.8514136250854792,
|
|
"rewards/rejected": -0.3806089527229743,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 10.605104446411133,
|
|
"kl": 12.012824058532715,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -33926131.2,
|
|
"logits/rejected": -35427110.4,
|
|
"logps/chosen": -435.0607421875,
|
|
"logps/rejected": -376.75380859375,
|
|
"loss": 0.5607,
|
|
"loss/base_kto": 3.1881015300750732,
|
|
"metrics/advantage_var": 526.0701293945312,
|
|
"regularization/mmd": 1.2978696823120117,
|
|
"regularization/rkhs_norm": 250.0712432861328,
|
|
"rewards/chosen": 0.4545284271240234,
|
|
"rewards/margins": 0.8710566520690918,
|
|
"rewards/rejected": -0.41652822494506836,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 13.993820190429688,
|
|
"kl": 15.309898376464844,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -33001932.2247191,
|
|
"logits/rejected": -35933534.68493151,
|
|
"logps/chosen": -511.3995786516854,
|
|
"logps/rejected": -389.74657534246575,
|
|
"loss": 0.578,
|
|
"loss/base_kto": 3.1738641262054443,
|
|
"metrics/advantage_var": 682.7782592773438,
|
|
"regularization/mmd": 1.4504328966140747,
|
|
"regularization/rkhs_norm": 279.4668273925781,
|
|
"rewards/chosen": 0.565710174903441,
|
|
"rewards/margins": 0.9253945776247356,
|
|
"rewards/rejected": -0.3596844027212947,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 35.92338562011719,
|
|
"kl": 13.133770942687988,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35608018.63291139,
|
|
"logits/rejected": -34120659.75308642,
|
|
"logps/chosen": -468.28352452531647,
|
|
"logps/rejected": -375.6628086419753,
|
|
"loss": 0.5892,
|
|
"loss/base_kto": 3.211897373199463,
|
|
"metrics/advantage_var": 1265.9241943359375,
|
|
"regularization/mmd": 1.5015980005264282,
|
|
"regularization/rkhs_norm": 289.3252258300781,
|
|
"rewards/chosen": 0.5198318384870698,
|
|
"rewards/margins": 0.8925337895022125,
|
|
"rewards/rejected": -0.37270195101514275,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 13.772028923034668,
|
|
"kl": 13.257220268249512,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34434898.49319214,
|
|
"logits/rejected": -36728699.65751211,
|
|
"logps/chosen": -472.11450453857793,
|
|
"logps/rejected": -378.4660238287561,
|
|
"loss": 0.5646,
|
|
"loss/base_kto": 3.2091243267059326,
|
|
"metrics/advantage_var": 542.9606323242188,
|
|
"regularization/mmd": 1.3076448440551758,
|
|
"regularization/rkhs_norm": 251.95469665527344,
|
|
"rewards/chosen": 0.5272039508675302,
|
|
"rewards/margins": 0.8476685210932597,
|
|
"rewards/rejected": -0.3204645702257295,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 16.19072723388672,
|
|
"kl": 19.78019905090332,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34483756.75259259,
|
|
"logits/rejected": -36526228.945454545,
|
|
"logps/chosen": -483.295,
|
|
"logps/rejected": -361.6900309917355,
|
|
"loss": 0.569,
|
|
"loss/base_kto": 3.24640154838562,
|
|
"metrics/advantage_var": 513.3463134765625,
|
|
"regularization/mmd": 1.3055371046066284,
|
|
"regularization/rkhs_norm": 251.54856872558594,
|
|
"rewards/chosen": 0.6207450810185186,
|
|
"rewards/margins": 0.8062072256022286,
|
|
"rewards/rejected": -0.18546214458370996,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 12.740949630737305,
|
|
"kl": 17.942371368408203,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35217482.58573596,
|
|
"logits/rejected": -36557126.492753625,
|
|
"logps/chosen": -485.2211684370258,
|
|
"logps/rejected": -382.9302033011272,
|
|
"loss": 0.5682,
|
|
"loss/base_kto": 3.2099609375,
|
|
"metrics/advantage_var": 600.2092895507812,
|
|
"regularization/mmd": 1.3359686136245728,
|
|
"regularization/rkhs_norm": 257.4120788574219,
|
|
"rewards/chosen": 0.5463747710487007,
|
|
"rewards/margins": 0.8105304553186534,
|
|
"rewards/rejected": -0.2641556842699527,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 12.415069580078125,
|
|
"kl": 12.111226081848145,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35118172.80491552,
|
|
"logits/rejected": -37141027.00158983,
|
|
"logps/chosen": -498.34317396313367,
|
|
"logps/rejected": -368.6554550874404,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.2402939796447754,
|
|
"metrics/advantage_var": 519.644287109375,
|
|
"regularization/mmd": 1.28346586227417,
|
|
"regularization/rkhs_norm": 247.29592895507812,
|
|
"rewards/chosen": 0.49320662772417434,
|
|
"rewards/margins": 0.8261065855720728,
|
|
"rewards/rejected": -0.33289995784789844,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 12.395271301269531,
|
|
"kl": 15.369041442871094,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34982349.43209877,
|
|
"logits/rejected": -35942944.405063294,
|
|
"logps/chosen": -478.2004726080247,
|
|
"logps/rejected": -357.84355221518985,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.234243154525757,
|
|
"metrics/advantage_var": 580.8305053710938,
|
|
"regularization/mmd": 1.3032277822494507,
|
|
"regularization/rkhs_norm": 251.10362243652344,
|
|
"rewards/chosen": 0.5746059653199749,
|
|
"rewards/margins": 0.8484069721682739,
|
|
"rewards/rejected": -0.27380100684829906,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 14.787806510925293,
|
|
"kl": 13.124091148376465,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35277166.518053375,
|
|
"logits/rejected": -36245905.31881804,
|
|
"logps/chosen": -470.9988716640502,
|
|
"logps/rejected": -370.8780375194401,
|
|
"loss": 0.5702,
|
|
"loss/base_kto": 3.224691390991211,
|
|
"metrics/advantage_var": 586.4566040039062,
|
|
"regularization/mmd": 1.3371353149414062,
|
|
"regularization/rkhs_norm": 257.6368713378906,
|
|
"rewards/chosen": 0.5041927415497449,
|
|
"rewards/margins": 0.8530821769811942,
|
|
"rewards/rejected": -0.34888943543144924,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 11.733684539794922,
|
|
"kl": 12.240571975708008,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34126317.571205005,
|
|
"logits/rejected": -36187818.13416537,
|
|
"logps/chosen": -498.07228090766824,
|
|
"logps/rejected": -394.8217141185647,
|
|
"loss": 0.572,
|
|
"loss/base_kto": 3.2093944549560547,
|
|
"metrics/advantage_var": 592.4822387695312,
|
|
"regularization/mmd": 1.3662270307540894,
|
|
"regularization/rkhs_norm": 263.2421875,
|
|
"rewards/chosen": 0.4863401601012324,
|
|
"rewards/margins": 0.8491955922872698,
|
|
"rewards/rejected": -0.36285543218603744,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 14.618077278137207,
|
|
"kl": 15.612887382507324,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -32535558.243902437,
|
|
"logits/rejected": -36130855.384615384,
|
|
"logps/chosen": -474.2106516768293,
|
|
"logps/rejected": -368.81961137820514,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.208326816558838,
|
|
"metrics/advantage_var": 494.7369079589844,
|
|
"regularization/mmd": 1.2807390689849854,
|
|
"regularization/rkhs_norm": 246.7705535888672,
|
|
"rewards/chosen": 0.6010621698891244,
|
|
"rewards/margins": 0.844801811518857,
|
|
"rewards/rejected": -0.24373964162973258,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 10.482880592346191,
|
|
"kl": 13.759737968444824,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34822329.02180685,
|
|
"logits/rejected": -35181417.12852664,
|
|
"logps/chosen": -463.89369158878503,
|
|
"logps/rejected": -371.80505485893417,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.190035343170166,
|
|
"metrics/advantage_var": 493.6685485839844,
|
|
"regularization/mmd": 1.284398078918457,
|
|
"regularization/rkhs_norm": 247.4755401611328,
|
|
"rewards/chosen": 0.5354623764846184,
|
|
"rewards/margins": 0.8865311900836339,
|
|
"rewards/rejected": -0.3510688135990155,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 13.44016170501709,
|
|
"kl": 11.417581558227539,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35275498.83954619,
|
|
"logits/rejected": -36027245.27300151,
|
|
"logps/chosen": -471.57232576985416,
|
|
"logps/rejected": -352.30453431372547,
|
|
"loss": 0.5642,
|
|
"loss/base_kto": 3.2011451721191406,
|
|
"metrics/advantage_var": 573.5377197265625,
|
|
"regularization/mmd": 1.3127566576004028,
|
|
"regularization/rkhs_norm": 252.9396514892578,
|
|
"rewards/chosen": 0.5109668404008813,
|
|
"rewards/margins": 0.8733932990214388,
|
|
"rewards/rejected": -0.3624264586205576,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 12.393445014953613,
|
|
"kl": 13.731520652770996,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35249347.047619045,
|
|
"logits/rejected": -36782729.05846154,
|
|
"logps/chosen": -499.02261904761906,
|
|
"logps/rejected": -386.26072115384613,
|
|
"loss": 0.5613,
|
|
"loss/base_kto": 3.190993070602417,
|
|
"metrics/advantage_var": 593.7273559570312,
|
|
"regularization/mmd": 1.2994486093521118,
|
|
"regularization/rkhs_norm": 250.3754425048828,
|
|
"rewards/chosen": 0.4655083550347222,
|
|
"rewards/margins": 0.8652361851879674,
|
|
"rewards/rejected": -0.3997278301532452,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 12.217032432556152,
|
|
"kl": 13.862927436828613,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35165086.95424837,
|
|
"logits/rejected": -35962671.52095808,
|
|
"logps/chosen": -485.632506127451,
|
|
"logps/rejected": -395.9298278443114,
|
|
"loss": 0.5668,
|
|
"loss/base_kto": 3.209890127182007,
|
|
"metrics/advantage_var": 554.9439697265625,
|
|
"regularization/mmd": 1.3247766494750977,
|
|
"regularization/rkhs_norm": 255.255615234375,
|
|
"rewards/chosen": 0.48636143503625406,
|
|
"rewards/margins": 0.8412285593082179,
|
|
"rewards/rejected": -0.3548671242719639,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 13.013198852539062,
|
|
"kl": 12.293803215026855,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34143723.13479624,
|
|
"logits/rejected": -35215666.24299066,
|
|
"logps/chosen": -496.9177605799373,
|
|
"logps/rejected": -375.9469918224299,
|
|
"loss": 0.5679,
|
|
"loss/base_kto": 3.2109673023223877,
|
|
"metrics/advantage_var": 595.9959716796875,
|
|
"regularization/mmd": 1.3321644067764282,
|
|
"regularization/rkhs_norm": 256.6791076660156,
|
|
"rewards/chosen": 0.457733584795626,
|
|
"rewards/margins": 0.8463017855933042,
|
|
"rewards/rejected": -0.3885682007976782,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 14.143969535827637,
|
|
"kl": 5.677832126617432,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34584572.78492936,
|
|
"logits/rejected": -35702545.11975116,
|
|
"logps/chosen": -499.6532574568289,
|
|
"logps/rejected": -374.1315124416796,
|
|
"loss": 0.5748,
|
|
"loss/base_kto": 3.2255847454071045,
|
|
"metrics/advantage_var": 607.7427978515625,
|
|
"regularization/mmd": 1.372963786125183,
|
|
"regularization/rkhs_norm": 264.5401916503906,
|
|
"rewards/chosen": 0.3632711413516729,
|
|
"rewards/margins": 0.8885416894113054,
|
|
"rewards/rejected": -0.5252705480596326,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 13.74710464477539,
|
|
"kl": 11.586774826049805,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -36909896.69135802,
|
|
"logits/rejected": -36242568.101265825,
|
|
"logps/chosen": -480.4585262345679,
|
|
"logps/rejected": -380.59745846518985,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.2076542377471924,
|
|
"metrics/advantage_var": 519.88232421875,
|
|
"regularization/mmd": 1.2929667234420776,
|
|
"regularization/rkhs_norm": 249.1265106201172,
|
|
"rewards/chosen": 0.5449497081615307,
|
|
"rewards/margins": 0.8521234250325749,
|
|
"rewards/rejected": -0.3071737168710443,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 13.282003402709961,
|
|
"kl": 12.539875030517578,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34782912.0,
|
|
"logits/rejected": -35688966.4,
|
|
"logps/chosen": -508.859130859375,
|
|
"logps/rejected": -363.0813720703125,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.2583274841308594,
|
|
"metrics/advantage_var": 459.13287353515625,
|
|
"regularization/mmd": 1.2167490720748901,
|
|
"regularization/rkhs_norm": 234.4410400390625,
|
|
"rewards/chosen": 0.4636064529418945,
|
|
"rewards/margins": 0.7895606994628905,
|
|
"rewards/rejected": -0.3259542465209961,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 13.528100967407227,
|
|
"kl": 9.498781204223633,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33393605.533442087,
|
|
"logits/rejected": -34550902.21289355,
|
|
"logps/chosen": -464.4634482055465,
|
|
"logps/rejected": -373.27857946026984,
|
|
"loss": 0.5653,
|
|
"loss/base_kto": 3.2615106105804443,
|
|
"metrics/advantage_var": 523.0314331054688,
|
|
"regularization/mmd": 1.2610514163970947,
|
|
"regularization/rkhs_norm": 242.97715759277344,
|
|
"rewards/chosen": 0.32341366841003516,
|
|
"rewards/margins": 0.7928392186964154,
|
|
"rewards/rejected": -0.4694255502863802,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 13.46378231048584,
|
|
"kl": 11.449333190917969,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33959516.22360248,
|
|
"logits/rejected": -35483899.16981132,
|
|
"logps/chosen": -461.790275621118,
|
|
"logps/rejected": -409.1107507861635,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.1564557552337646,
|
|
"metrics/advantage_var": 561.4967041015625,
|
|
"regularization/mmd": 1.3471063375473022,
|
|
"regularization/rkhs_norm": 259.5580749511719,
|
|
"rewards/chosen": 0.452808830308618,
|
|
"rewards/margins": 0.9065669740874849,
|
|
"rewards/rejected": -0.4537581437788669,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 12.276199340820312,
|
|
"kl": 18.980093002319336,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34802897.41706924,
|
|
"logits/rejected": -36302767.19878604,
|
|
"logps/chosen": -483.919384057971,
|
|
"logps/rejected": -393.4671614188164,
|
|
"loss": 0.5591,
|
|
"loss/base_kto": 3.1546790599823,
|
|
"metrics/advantage_var": 538.6190795898438,
|
|
"regularization/mmd": 1.3179327249526978,
|
|
"regularization/rkhs_norm": 253.93692016601562,
|
|
"rewards/chosen": 0.561616661084063,
|
|
"rewards/margins": 0.8924342237604075,
|
|
"rewards/rejected": -0.3308175626763444,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 11.4161958694458,
|
|
"kl": 12.60518741607666,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33908111.02347418,
|
|
"logits/rejected": -34133910.56474259,
|
|
"logps/chosen": -465.4889475743349,
|
|
"logps/rejected": -362.661880850234,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.2447450160980225,
|
|
"metrics/advantage_var": 490.63330078125,
|
|
"regularization/mmd": 1.2534472942352295,
|
|
"regularization/rkhs_norm": 241.51199340820312,
|
|
"rewards/chosen": 0.4799577357810055,
|
|
"rewards/margins": 0.805216892387504,
|
|
"rewards/rejected": -0.3252591566064986,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 13.910622596740723,
|
|
"kl": 12.987527847290039,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33991888.103225805,
|
|
"logits/rejected": -35400443.345454544,
|
|
"logps/chosen": -486.30120967741937,
|
|
"logps/rejected": -373.7405066287879,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.1726856231689453,
|
|
"metrics/advantage_var": 560.0360717773438,
|
|
"regularization/mmd": 1.344662070274353,
|
|
"regularization/rkhs_norm": 259.0871276855469,
|
|
"rewards/chosen": 0.4919520716513357,
|
|
"rewards/margins": 0.8867113711896879,
|
|
"rewards/rejected": -0.39475929953835226,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 9.518303871154785,
|
|
"kl": 10.206445693969727,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35678368.38554217,
|
|
"logits/rejected": -34778451.116883114,
|
|
"logps/chosen": -488.46169051204816,
|
|
"logps/rejected": -385.71218039772725,
|
|
"loss": 0.5571,
|
|
"loss/base_kto": 3.225837469100952,
|
|
"metrics/advantage_var": 455.4820251464844,
|
|
"regularization/mmd": 1.230714201927185,
|
|
"regularization/rkhs_norm": 237.1318359375,
|
|
"rewards/chosen": 0.45567735419215927,
|
|
"rewards/margins": 0.8314866149692239,
|
|
"rewards/rejected": -0.3758092607770647,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 9.84069538116455,
|
|
"kl": 15.96777057647705,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34350347.61815336,
|
|
"logits/rejected": -35676370.87051482,
|
|
"logps/chosen": -483.6515062597809,
|
|
"logps/rejected": -367.29207293291734,
|
|
"loss": 0.5522,
|
|
"loss/base_kto": 3.1634433269500732,
|
|
"metrics/advantage_var": 501.04638671875,
|
|
"regularization/mmd": 1.2541414499282837,
|
|
"regularization/rkhs_norm": 241.645751953125,
|
|
"rewards/chosen": 0.6091435630929676,
|
|
"rewards/margins": 0.9014785377419535,
|
|
"rewards/rejected": -0.29233497464898595,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 13.790901184082031,
|
|
"kl": 11.686732292175293,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35776431.075987846,
|
|
"logits/rejected": -35465413.5562701,
|
|
"logps/chosen": -479.35961246200606,
|
|
"logps/rejected": -377.39353898713824,
|
|
"loss": 0.5659,
|
|
"loss/base_kto": 3.223767042160034,
|
|
"metrics/advantage_var": 507.0044860839844,
|
|
"regularization/mmd": 1.3035944700241089,
|
|
"regularization/rkhs_norm": 251.1742706298828,
|
|
"rewards/chosen": 0.5425103370179521,
|
|
"rewards/margins": 0.8425677512782768,
|
|
"rewards/rejected": -0.30005741426032456,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 12.626648902893066,
|
|
"kl": 9.487886428833008,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35338563.91836735,
|
|
"logits/rejected": -35923864.41618497,
|
|
"logps/chosen": -465.55575042517006,
|
|
"logps/rejected": -378.18768063583815,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.291943073272705,
|
|
"metrics/advantage_var": 445.0700378417969,
|
|
"regularization/mmd": 1.2204208374023438,
|
|
"regularization/rkhs_norm": 235.14854431152344,
|
|
"rewards/chosen": 0.35613624417051976,
|
|
"rewards/margins": 0.7489256107878879,
|
|
"rewards/rejected": -0.39278936661736813,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 12.45100212097168,
|
|
"kl": 11.316283226013184,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35040368.3902439,
|
|
"logits/rejected": -36118219.48717949,
|
|
"logps/chosen": -495.841034679878,
|
|
"logps/rejected": -358.8885967548077,
|
|
"loss": 0.5664,
|
|
"loss/base_kto": 3.2071197032928467,
|
|
"metrics/advantage_var": 580.8953857421875,
|
|
"regularization/mmd": 1.3240996599197388,
|
|
"regularization/rkhs_norm": 255.1251678466797,
|
|
"rewards/chosen": 0.4648424474204459,
|
|
"rewards/margins": 0.8998550202713824,
|
|
"rewards/rejected": -0.4350125728509365,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 12.758334159851074,
|
|
"kl": 9.478241920471191,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34427510.15384615,
|
|
"logits/rejected": -36078775.09933775,
|
|
"logps/chosen": -484.60022189349115,
|
|
"logps/rejected": -371.0925858857616,
|
|
"loss": 0.5688,
|
|
"loss/base_kto": 3.214158773422241,
|
|
"metrics/advantage_var": 529.6085815429688,
|
|
"regularization/mmd": 1.3360570669174194,
|
|
"regularization/rkhs_norm": 257.42913818359375,
|
|
"rewards/chosen": 0.5376529129299187,
|
|
"rewards/margins": 0.8611009930306792,
|
|
"rewards/rejected": -0.32344808010076054,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 9.090041160583496,
|
|
"kl": 14.543585777282715,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34697005.42620232,
|
|
"logits/rejected": -36461869.89037037,
|
|
"logps/chosen": -481.5368470149254,
|
|
"logps/rejected": -361.2715277777778,
|
|
"loss": 0.5324,
|
|
"loss/base_kto": 3.1204822063446045,
|
|
"metrics/advantage_var": 386.4324645996094,
|
|
"regularization/mmd": 1.139040231704712,
|
|
"regularization/rkhs_norm": 219.46824645996094,
|
|
"rewards/chosen": 0.5331376868100902,
|
|
"rewards/margins": 0.8890228096037823,
|
|
"rewards/rejected": -0.3558851227936921,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 10.383761405944824,
|
|
"kl": 5.766092777252197,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34437413.5136876,
|
|
"logits/rejected": -35147844.370257966,
|
|
"logps/chosen": -465.67340982286635,
|
|
"logps/rejected": -376.12836684370257,
|
|
"loss": 0.5261,
|
|
"loss/base_kto": 3.1161365509033203,
|
|
"metrics/advantage_var": 337.5785217285156,
|
|
"regularization/mmd": 1.0925873517990112,
|
|
"regularization/rkhs_norm": 210.5177764892578,
|
|
"rewards/chosen": 0.40059321552467037,
|
|
"rewards/margins": 0.9246520886100744,
|
|
"rewards/rejected": -0.524058873085404,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 11.793994903564453,
|
|
"kl": 10.391288757324219,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33245288.3566879,
|
|
"logits/rejected": -35781223.65644172,
|
|
"logps/chosen": -458.1954617834395,
|
|
"logps/rejected": -379.5230061349693,
|
|
"loss": 0.5306,
|
|
"loss/base_kto": 3.122455596923828,
|
|
"metrics/advantage_var": 356.0881652832031,
|
|
"regularization/mmd": 1.1224075555801392,
|
|
"regularization/rkhs_norm": 216.26353454589844,
|
|
"rewards/chosen": 0.48445518153488254,
|
|
"rewards/margins": 0.8950714887057271,
|
|
"rewards/rejected": -0.4106163071708445,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 7.086558818817139,
|
|
"kl": 9.862372398376465,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34235303.23219814,
|
|
"logits/rejected": -36231839.899053626,
|
|
"logps/chosen": -483.79939047987614,
|
|
"logps/rejected": -378.72279179810727,
|
|
"loss": 0.529,
|
|
"loss/base_kto": 3.0944149494171143,
|
|
"metrics/advantage_var": 362.52410888671875,
|
|
"regularization/mmd": 1.1374865770339966,
|
|
"regularization/rkhs_norm": 219.1688995361328,
|
|
"rewards/chosen": 0.45048064713138547,
|
|
"rewards/margins": 0.9434058638543251,
|
|
"rewards/rejected": -0.49292521672293965,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 8.81950855255127,
|
|
"kl": 12.163896560668945,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33259309.774834435,
|
|
"logits/rejected": -34815491.0295858,
|
|
"logps/chosen": -451.4697330298013,
|
|
"logps/rejected": -374.1150147928994,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.0771431922912598,
|
|
"metrics/advantage_var": 350.685302734375,
|
|
"regularization/mmd": 1.1020879745483398,
|
|
"regularization/rkhs_norm": 212.34837341308594,
|
|
"rewards/chosen": 0.49697593032129556,
|
|
"rewards/margins": 0.9272090448911458,
|
|
"rewards/rejected": -0.4302331145698502,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 11.56906795501709,
|
|
"kl": 9.059837341308594,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34197531.34718101,
|
|
"logits/rejected": -35774173.35973597,
|
|
"logps/chosen": -467.30777077151333,
|
|
"logps/rejected": -372.39248143564356,
|
|
"loss": 0.5375,
|
|
"loss/base_kto": 3.1023190021514893,
|
|
"metrics/advantage_var": 423.91339111328125,
|
|
"regularization/mmd": 1.197938084602356,
|
|
"regularization/rkhs_norm": 230.8165740966797,
|
|
"rewards/chosen": 0.5261493082923776,
|
|
"rewards/margins": 0.9705709568781347,
|
|
"rewards/rejected": -0.444421648585757,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 9.984126091003418,
|
|
"kl": 15.941340446472168,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33585391.13772455,
|
|
"logits/rejected": -34798203.81699347,
|
|
"logps/chosen": -469.1608345808383,
|
|
"logps/rejected": -356.4097732843137,
|
|
"loss": 0.524,
|
|
"loss/base_kto": 3.0616936683654785,
|
|
"metrics/advantage_var": 350.7194519042969,
|
|
"regularization/mmd": 1.1301138401031494,
|
|
"regularization/rkhs_norm": 217.74832153320312,
|
|
"rewards/chosen": 0.6396038946277367,
|
|
"rewards/margins": 0.939726249172582,
|
|
"rewards/rejected": -0.30012235454484526,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 12.476960182189941,
|
|
"kl": 17.748157501220703,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34125396.673879445,
|
|
"logits/rejected": -33902740.82780411,
|
|
"logps/chosen": -474.78390649149924,
|
|
"logps/rejected": -367.66641982622434,
|
|
"loss": 0.5231,
|
|
"loss/base_kto": 3.081040143966675,
|
|
"metrics/advantage_var": 352.6298828125,
|
|
"regularization/mmd": 1.1038602590560913,
|
|
"regularization/rkhs_norm": 212.6898193359375,
|
|
"rewards/chosen": 0.5997229551052936,
|
|
"rewards/margins": 0.9045915357110401,
|
|
"rewards/rejected": -0.30486858060574645,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 11.377838134765625,
|
|
"kl": 16.43869972229004,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33773353.44761905,
|
|
"logits/rejected": -36203743.704615384,
|
|
"logps/chosen": -470.503373015873,
|
|
"logps/rejected": -352.26057692307694,
|
|
"loss": 0.5219,
|
|
"loss/base_kto": 3.077666997909546,
|
|
"metrics/advantage_var": 353.3399353027344,
|
|
"regularization/mmd": 1.0976718664169312,
|
|
"regularization/rkhs_norm": 211.49746704101562,
|
|
"rewards/chosen": 0.6101555718315972,
|
|
"rewards/margins": 0.9196656003772703,
|
|
"rewards/rejected": -0.30951002854567305,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 10.534581184387207,
|
|
"kl": 11.540972709655762,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35385649.02127659,
|
|
"logits/rejected": -35417874.93247589,
|
|
"logps/chosen": -509.9543123100304,
|
|
"logps/rejected": -367.4504370980707,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.0438456535339355,
|
|
"metrics/advantage_var": 392.5074462890625,
|
|
"regularization/mmd": 1.1931309700012207,
|
|
"regularization/rkhs_norm": 229.89036560058594,
|
|
"rewards/chosen": 0.6179606428986987,
|
|
"rewards/margins": 0.9905718328605154,
|
|
"rewards/rejected": -0.37261118996181675,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 8.50450325012207,
|
|
"kl": 12.761210441589355,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34851502.77293233,
|
|
"logits/rejected": -35710290.00325203,
|
|
"logps/chosen": -478.3872180451128,
|
|
"logps/rejected": -370.5042174796748,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.072681427001953,
|
|
"metrics/advantage_var": 397.5965270996094,
|
|
"regularization/mmd": 1.1549023389816284,
|
|
"regularization/rkhs_norm": 222.5245361328125,
|
|
"rewards/chosen": 0.5885087321575423,
|
|
"rewards/margins": 0.9599269609272323,
|
|
"rewards/rejected": -0.37141822876969005,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 10.313164710998535,
|
|
"kl": 14.244941711425781,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -33766288.73070866,
|
|
"logits/rejected": -35691242.170542635,
|
|
"logps/chosen": -485.36678149606297,
|
|
"logps/rejected": -370.7001937984496,
|
|
"loss": 0.526,
|
|
"loss/base_kto": 3.032137632369995,
|
|
"metrics/advantage_var": 391.1866149902344,
|
|
"regularization/mmd": 1.1759032011032104,
|
|
"regularization/rkhs_norm": 226.5709686279297,
|
|
"rewards/chosen": 0.6140625961183563,
|
|
"rewards/margins": 0.9991652865858951,
|
|
"rewards/rejected": -0.38510269046753876,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 10.61887264251709,
|
|
"kl": 14.920964241027832,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35428939.48717949,
|
|
"logits/rejected": -36445390.048780486,
|
|
"logps/chosen": -470.86758814102564,
|
|
"logps/rejected": -389.7338509908537,
|
|
"loss": 0.5285,
|
|
"loss/base_kto": 3.0771355628967285,
|
|
"metrics/advantage_var": 377.6502380371094,
|
|
"regularization/mmd": 1.1507731676101685,
|
|
"regularization/rkhs_norm": 221.7289581298828,
|
|
"rewards/chosen": 0.5433956048427484,
|
|
"rewards/margins": 0.9460372733950541,
|
|
"rewards/rejected": -0.40264166855230565,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 14.334951400756836,
|
|
"kl": 12.795031547546387,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33141407.744,
|
|
"logits/rejected": -35409313.41679389,
|
|
"logps/chosen": -491.1893,
|
|
"logps/rejected": -383.1791507633588,
|
|
"loss": 0.5253,
|
|
"loss/base_kto": 3.0389838218688965,
|
|
"metrics/advantage_var": 382.8691101074219,
|
|
"regularization/mmd": 1.1637873649597168,
|
|
"regularization/rkhs_norm": 224.23648071289062,
|
|
"rewards/chosen": 0.549120361328125,
|
|
"rewards/margins": 0.9864840339485925,
|
|
"rewards/rejected": -0.43736367262046755,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 8.612019538879395,
|
|
"kl": 13.4284029006958,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34413065.42331288,
|
|
"logits/rejected": -36131112.76433121,
|
|
"logps/chosen": -481.9517350460123,
|
|
"logps/rejected": -391.4028662420382,
|
|
"loss": 0.5287,
|
|
"loss/base_kto": 3.153485059738159,
|
|
"metrics/advantage_var": 325.0394592285156,
|
|
"regularization/mmd": 1.075948715209961,
|
|
"regularization/rkhs_norm": 207.3118896484375,
|
|
"rewards/chosen": 0.5054645655345331,
|
|
"rewards/margins": 0.8459768108585404,
|
|
"rewards/rejected": -0.34051224532400726,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 9.022269248962402,
|
|
"kl": 14.502161026000977,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35471320.14371257,
|
|
"logits/rejected": -35742746.77124183,
|
|
"logps/chosen": -462.3091317365269,
|
|
"logps/rejected": -397.0358455882353,
|
|
"loss": 0.5298,
|
|
"loss/base_kto": 3.050593852996826,
|
|
"metrics/advantage_var": 399.081298828125,
|
|
"regularization/mmd": 1.187691569328308,
|
|
"regularization/rkhs_norm": 228.84228515625,
|
|
"rewards/chosen": 0.5995275074850299,
|
|
"rewards/margins": 0.969992181493468,
|
|
"rewards/rejected": -0.370464674008438,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 16.58113670349121,
|
|
"kl": 17.321325302124023,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35684517.11007752,
|
|
"logits/rejected": -35488214.878740154,
|
|
"logps/chosen": -492.69234496124034,
|
|
"logps/rejected": -362.6099409448819,
|
|
"loss": 0.5299,
|
|
"loss/base_kto": 3.0663952827453613,
|
|
"metrics/advantage_var": 400.5253601074219,
|
|
"regularization/mmd": 1.1730730533599854,
|
|
"regularization/rkhs_norm": 226.025634765625,
|
|
"rewards/chosen": 0.6765439528827519,
|
|
"rewards/margins": 0.9499858743271417,
|
|
"rewards/rejected": -0.2734419214443898,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 11.718950271606445,
|
|
"kl": 13.231635093688965,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34744012.030075185,
|
|
"logits/rejected": -34855761.17073171,
|
|
"logps/chosen": -488.3638627819549,
|
|
"logps/rejected": -396.16900406504067,
|
|
"loss": 0.5295,
|
|
"loss/base_kto": 3.0555837154388428,
|
|
"metrics/advantage_var": 399.5165100097656,
|
|
"regularization/mmd": 1.1801179647445679,
|
|
"regularization/rkhs_norm": 227.383056640625,
|
|
"rewards/chosen": 0.5781597854499531,
|
|
"rewards/margins": 0.9702725297890664,
|
|
"rewards/rejected": -0.3921127443391133,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 11.017359733581543,
|
|
"kl": 14.590458869934082,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -32775911.277258568,
|
|
"logits/rejected": -36324711.52351097,
|
|
"logps/chosen": -503.3436526479751,
|
|
"logps/rejected": -360.70197394200625,
|
|
"loss": 0.527,
|
|
"loss/base_kto": 3.0754008293151855,
|
|
"metrics/advantage_var": 374.4561462402344,
|
|
"regularization/mmd": 1.1406577825546265,
|
|
"regularization/rkhs_norm": 219.77993774414062,
|
|
"rewards/chosen": 0.5987370095891744,
|
|
"rewards/margins": 0.9385808552555635,
|
|
"rewards/rejected": -0.3398438456663891,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 10.031533241271973,
|
|
"kl": 14.779060363769531,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34274614.01834863,
|
|
"logits/rejected": -35749269.6741214,
|
|
"logps/chosen": -459.78841743119267,
|
|
"logps/rejected": -377.9798572284345,
|
|
"loss": 0.5299,
|
|
"loss/base_kto": 3.0682921409606934,
|
|
"metrics/advantage_var": 395.0757141113281,
|
|
"regularization/mmd": 1.1708500385284424,
|
|
"regularization/rkhs_norm": 225.5972900390625,
|
|
"rewards/chosen": 0.6174478326733324,
|
|
"rewards/margins": 0.9718213623695684,
|
|
"rewards/rejected": -0.35437352969623603,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 13.739367485046387,
|
|
"kl": 14.65582275390625,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34720911.07064364,
|
|
"logits/rejected": -35703284.05598756,
|
|
"logps/chosen": -484.7956240188383,
|
|
"logps/rejected": -396.47390163297047,
|
|
"loss": 0.5281,
|
|
"loss/base_kto": 3.1156222820281982,
|
|
"metrics/advantage_var": 356.6663513183594,
|
|
"regularization/mmd": 1.1090455055236816,
|
|
"regularization/rkhs_norm": 213.6889190673828,
|
|
"rewards/chosen": 0.5509815066351059,
|
|
"rewards/margins": 0.891159360042899,
|
|
"rewards/rejected": -0.34017785340779305,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 10.554007530212402,
|
|
"kl": 13.29034423828125,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34964727.94968554,
|
|
"logits/rejected": -35922994.88198758,
|
|
"logps/chosen": -475.53016902515725,
|
|
"logps/rejected": -404.9791828416149,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.0745606422424316,
|
|
"metrics/advantage_var": 430.95556640625,
|
|
"regularization/mmd": 1.1665340662002563,
|
|
"regularization/rkhs_norm": 224.7657012939453,
|
|
"rewards/chosen": 0.5371469941529088,
|
|
"rewards/margins": 0.9245654686227749,
|
|
"rewards/rejected": -0.38741847446986605,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 10.040180206298828,
|
|
"kl": 12.796968460083008,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -32839978.59904913,
|
|
"logits/rejected": -35519508.51155624,
|
|
"logps/chosen": -509.7398969889065,
|
|
"logps/rejected": -374.0168287750385,
|
|
"loss": 0.5265,
|
|
"loss/base_kto": 3.043978452682495,
|
|
"metrics/advantage_var": 385.7436218261719,
|
|
"regularization/mmd": 1.1682690382003784,
|
|
"regularization/rkhs_norm": 225.10000610351562,
|
|
"rewards/chosen": 0.608874337608954,
|
|
"rewards/margins": 0.9669892531716586,
|
|
"rewards/rejected": -0.35811491556270464,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 9.16383171081543,
|
|
"kl": 9.79554271697998,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34649568.46920821,
|
|
"logits/rejected": -35457599.35785953,
|
|
"logps/chosen": -480.17723607038124,
|
|
"logps/rejected": -382.42605560200667,
|
|
"loss": 0.5269,
|
|
"loss/base_kto": 3.112825870513916,
|
|
"metrics/advantage_var": 340.6751403808594,
|
|
"regularization/mmd": 1.1027467250823975,
|
|
"regularization/rkhs_norm": 212.4752960205078,
|
|
"rewards/chosen": 0.5558217963165552,
|
|
"rewards/margins": 0.9192394568288096,
|
|
"rewards/rejected": -0.3634176605122544,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.72305965423584,
|
|
"kl": 14.54590892791748,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34854229.333333336,
|
|
"logits/rejected": -36668520.94409938,
|
|
"logps/chosen": -497.75108097484275,
|
|
"logps/rejected": -390.07603843167703,
|
|
"loss": 0.5384,
|
|
"loss/base_kto": 3.0823018550872803,
|
|
"metrics/advantage_var": 432.26123046875,
|
|
"regularization/mmd": 1.2246301174163818,
|
|
"regularization/rkhs_norm": 235.9595489501953,
|
|
"rewards/chosen": 0.5804522532337116,
|
|
"rewards/margins": 0.9573801680604779,
|
|
"rewards/rejected": -0.3769279148267663,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 11.256319999694824,
|
|
"kl": 14.031867980957031,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34613001.05466238,
|
|
"logits/rejected": -37503807.027355626,
|
|
"logps/chosen": -509.44453376205786,
|
|
"logps/rejected": -336.250546162614,
|
|
"loss": 0.5275,
|
|
"loss/base_kto": 3.0460011959075928,
|
|
"metrics/advantage_var": 388.61212158203125,
|
|
"regularization/mmd": 1.1741565465927124,
|
|
"regularization/rkhs_norm": 226.234375,
|
|
"rewards/chosen": 0.6302729664891479,
|
|
"rewards/margins": 0.9777478966233732,
|
|
"rewards/rejected": -0.3474749301342254,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 10.791040420532227,
|
|
"kl": 11.97043514251709,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34284672.0,
|
|
"logits/rejected": -35075724.8,
|
|
"logps/chosen": -462.667578125,
|
|
"logps/rejected": -384.8657958984375,
|
|
"loss": 0.5237,
|
|
"loss/base_kto": 3.0678415298461914,
|
|
"metrics/advantage_var": 363.6769104003906,
|
|
"regularization/mmd": 1.1215388774871826,
|
|
"regularization/rkhs_norm": 216.0961151123047,
|
|
"rewards/chosen": 0.6000361442565918,
|
|
"rewards/margins": 0.9610976457595826,
|
|
"rewards/rejected": -0.3610615015029907,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 13.231522560119629,
|
|
"kl": 12.22443675994873,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34146268.46687697,
|
|
"logits/rejected": -35900609.3869969,
|
|
"logps/chosen": -488.47954455835963,
|
|
"logps/rejected": -373.31196787925694,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.0673561096191406,
|
|
"metrics/advantage_var": 374.9820251464844,
|
|
"regularization/mmd": 1.12788987159729,
|
|
"regularization/rkhs_norm": 217.31982421875,
|
|
"rewards/chosen": 0.5576446725743051,
|
|
"rewards/margins": 0.9636198254078481,
|
|
"rewards/rejected": -0.40597515283354296,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 11.246485710144043,
|
|
"kl": 10.425522804260254,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34195608.935064934,
|
|
"logits/rejected": -35350845.686746985,
|
|
"logps/chosen": -470.9198965097403,
|
|
"logps/rejected": -366.1610033885542,
|
|
"loss": 0.5233,
|
|
"loss/base_kto": 3.081230878829956,
|
|
"metrics/advantage_var": 337.57794189453125,
|
|
"regularization/mmd": 1.1048914194107056,
|
|
"regularization/rkhs_norm": 212.88853454589844,
|
|
"rewards/chosen": 0.5228175373820515,
|
|
"rewards/margins": 0.9380955470035452,
|
|
"rewards/rejected": -0.4152780096214938,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 10.961644172668457,
|
|
"kl": 9.417128562927246,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34885925.46341463,
|
|
"logits/rejected": -36548627.692307696,
|
|
"logps/chosen": -477.7841558689024,
|
|
"logps/rejected": -385.6079727564103,
|
|
"loss": 0.5327,
|
|
"loss/base_kto": 3.1277520656585693,
|
|
"metrics/advantage_var": 377.98529052734375,
|
|
"regularization/mmd": 1.1342254877090454,
|
|
"regularization/rkhs_norm": 218.54055786132812,
|
|
"rewards/chosen": 0.4911082197980183,
|
|
"rewards/margins": 0.9023305309646348,
|
|
"rewards/rejected": -0.41122231116661656,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 9.087610244750977,
|
|
"kl": 13.236366271972656,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34406472.437299035,
|
|
"logits/rejected": -34965463.53799392,
|
|
"logps/chosen": -501.6039991961415,
|
|
"logps/rejected": -387.48755699088144,
|
|
"loss": 0.5358,
|
|
"loss/base_kto": 3.1003830432891846,
|
|
"metrics/advantage_var": 432.23931884765625,
|
|
"regularization/mmd": 1.1862163543701172,
|
|
"regularization/rkhs_norm": 228.5580596923828,
|
|
"rewards/chosen": 0.5211313131154541,
|
|
"rewards/margins": 0.9342506437100591,
|
|
"rewards/rejected": -0.41311933059460487,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 6.814465045928955,
|
|
"kl": 11.84898853302002,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34181366.138674885,
|
|
"logits/rejected": -36011649.01426307,
|
|
"logps/chosen": -444.82612673343607,
|
|
"logps/rejected": -367.62418284469095,
|
|
"loss": 0.5253,
|
|
"loss/base_kto": 3.113685369491577,
|
|
"metrics/advantage_var": 341.3646240234375,
|
|
"regularization/mmd": 1.0885260105133057,
|
|
"regularization/rkhs_norm": 209.73526000976562,
|
|
"rewards/chosen": 0.5210508749187451,
|
|
"rewards/margins": 0.9039431510599768,
|
|
"rewards/rejected": -0.38289227614123167,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 11.534289360046387,
|
|
"kl": 10.679969787597656,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35391599.87860394,
|
|
"logits/rejected": -36157430.10628019,
|
|
"logps/chosen": -505.2507587253414,
|
|
"logps/rejected": -374.8183373590982,
|
|
"loss": 0.5279,
|
|
"loss/base_kto": 3.095287322998047,
|
|
"metrics/advantage_var": 374.14031982421875,
|
|
"regularization/mmd": 1.1282075643539429,
|
|
"regularization/rkhs_norm": 217.3810272216797,
|
|
"rewards/chosen": 0.536602014475057,
|
|
"rewards/margins": 0.9438256038454651,
|
|
"rewards/rejected": -0.4072235893704081,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 12.044182777404785,
|
|
"kl": 11.639398574829102,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33645047.01754386,
|
|
"logits/rejected": -34983496.91883614,
|
|
"logps/chosen": -476.5754585326954,
|
|
"logps/rejected": -369.31484494640125,
|
|
"loss": 0.5269,
|
|
"loss/base_kto": 3.0718960762023926,
|
|
"metrics/advantage_var": 381.8918151855469,
|
|
"regularization/mmd": 1.1435842514038086,
|
|
"regularization/rkhs_norm": 220.3437957763672,
|
|
"rewards/chosen": 0.5401317833713367,
|
|
"rewards/margins": 0.9533396042461453,
|
|
"rewards/rejected": -0.41320782087480856,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 12.774967193603516,
|
|
"kl": 12.074213981628418,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35974496.20382166,
|
|
"logits/rejected": -36406089.81595092,
|
|
"logps/chosen": -482.8093152866242,
|
|
"logps/rejected": -396.36838573619633,
|
|
"loss": 0.5312,
|
|
"loss/base_kto": 3.1223459243774414,
|
|
"metrics/advantage_var": 365.4447937011719,
|
|
"regularization/mmd": 1.1269159317016602,
|
|
"regularization/rkhs_norm": 217.13218688964844,
|
|
"rewards/chosen": 0.5236535527903563,
|
|
"rewards/margins": 0.9103144504490028,
|
|
"rewards/rejected": -0.3866608976586465,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.84286117553711,
|
|
"kl": 11.573938369750977,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33733320.21020093,
|
|
"logits/rejected": -36330698.211690366,
|
|
"logps/chosen": -480.12301970633695,
|
|
"logps/rejected": -389.0956753554502,
|
|
"loss": 0.5204,
|
|
"loss/base_kto": 3.0563924312591553,
|
|
"metrics/advantage_var": 355.6333312988281,
|
|
"regularization/mmd": 1.1071523427963257,
|
|
"regularization/rkhs_norm": 213.32412719726562,
|
|
"rewards/chosen": 0.5344630717494204,
|
|
"rewards/margins": 0.9534984424420547,
|
|
"rewards/rejected": -0.4190353706926343,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 13.87558650970459,
|
|
"kl": 11.596927642822266,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -33972182.786377706,
|
|
"logits/rejected": -36055750.662460566,
|
|
"logps/chosen": -479.59597523219816,
|
|
"logps/rejected": -364.93730283911674,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.097254514694214,
|
|
"metrics/advantage_var": 388.12646484375,
|
|
"regularization/mmd": 1.129944920539856,
|
|
"regularization/rkhs_norm": 217.7157745361328,
|
|
"rewards/chosen": 0.5392871774017995,
|
|
"rewards/margins": 0.9249849363140876,
|
|
"rewards/rejected": -0.385697758912288,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 11.813056945800781,
|
|
"kl": 11.99323558807373,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33321006.394822005,
|
|
"logits/rejected": -35000542.74320242,
|
|
"logps/chosen": -500.0081917475728,
|
|
"logps/rejected": -377.77922016616316,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.078052520751953,
|
|
"metrics/advantage_var": 396.63751220703125,
|
|
"regularization/mmd": 1.1525338888168335,
|
|
"regularization/rkhs_norm": 222.0681610107422,
|
|
"rewards/chosen": 0.5652463845064725,
|
|
"rewards/margins": 0.9629498952634117,
|
|
"rewards/rejected": -0.3977035107569392,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_loss": 0.558508157318326,
|
|
"train_runtime": 11054.4883,
|
|
"train_samples_per_second": 13.408,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|