f7b1975174
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed6 Source: Original Platform
2229 lines
80 KiB
JSON
2229 lines
80 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 15.022944450378418,
|
|
"kl": 13.015869140625,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37765250.03174603,
|
|
"logits/rejected": -37830841.89538462,
|
|
"logps/chosen": -492.9030257936508,
|
|
"logps/rejected": -370.1293269230769,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.9208168983459473,
|
|
"metrics/advantage_var": 233.6092071533203,
|
|
"regularization/mmd": 0.8639454245567322,
|
|
"regularization/rkhs_norm": 166.46348571777344,
|
|
"rewards/chosen": 0.09404822455512153,
|
|
"rewards/margins": 0.0626547596597264,
|
|
"rewards/rejected": 0.03139346489539513,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 17.054180145263672,
|
|
"kl": 7.279256343841553,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36347563.197511666,
|
|
"logits/rejected": -37333770.04709576,
|
|
"logps/chosen": -500.01895412130636,
|
|
"logps/rejected": -376.9522664835165,
|
|
"loss": 0.5814,
|
|
"loss/base_kto": 3.8649842739105225,
|
|
"metrics/advantage_var": 177.0762176513672,
|
|
"regularization/mmd": 0.7862296104431152,
|
|
"regularization/rkhs_norm": 151.4893341064453,
|
|
"rewards/chosen": 0.10241522425626337,
|
|
"rewards/margins": 0.12426469033758021,
|
|
"rewards/rejected": -0.02184946608131684,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 19.786029815673828,
|
|
"kl": 10.351292610168457,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36020624.83180428,
|
|
"logits/rejected": -36707154.60702875,
|
|
"logps/chosen": -474.0603975535168,
|
|
"logps/rejected": -395.8020666932907,
|
|
"loss": 0.5844,
|
|
"loss/base_kto": 3.8749122619628906,
|
|
"metrics/advantage_var": 184.5719451904297,
|
|
"regularization/mmd": 0.8005467653274536,
|
|
"regularization/rkhs_norm": 154.2479248046875,
|
|
"rewards/chosen": 0.17589327610960795,
|
|
"rewards/margins": 0.10272494504868043,
|
|
"rewards/rejected": 0.07316833106092752,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 15.770626068115234,
|
|
"kl": 4.177794456481934,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36548598.293838866,
|
|
"logits/rejected": -39029669.786707886,
|
|
"logps/chosen": -479.7108017377567,
|
|
"logps/rejected": -390.0850560278207,
|
|
"loss": 0.5874,
|
|
"loss/base_kto": 3.8683526515960693,
|
|
"metrics/advantage_var": 205.47239685058594,
|
|
"regularization/mmd": 0.8306226134300232,
|
|
"regularization/rkhs_norm": 160.0428924560547,
|
|
"rewards/chosen": 0.0643233590028109,
|
|
"rewards/margins": 0.12862897844455523,
|
|
"rewards/rejected": -0.06430561944174434,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 15.51358699798584,
|
|
"kl": 6.871208190917969,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -37117447.279620856,
|
|
"logits/rejected": -37090425.867078826,
|
|
"logps/chosen": -472.0468009478673,
|
|
"logps/rejected": -379.506472179289,
|
|
"loss": 0.5868,
|
|
"loss/base_kto": 3.862506866455078,
|
|
"metrics/advantage_var": 195.0583953857422,
|
|
"regularization/mmd": 0.831774890422821,
|
|
"regularization/rkhs_norm": 160.2649383544922,
|
|
"rewards/chosen": 0.0839598717471058,
|
|
"rewards/margins": 0.12648410928404763,
|
|
"rewards/rejected": -0.04252423753694184,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 20.489465713500977,
|
|
"kl": 10.069735527038574,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -38317105.498489425,
|
|
"logits/rejected": -38494121.8381877,
|
|
"logps/chosen": -477.451000755287,
|
|
"logps/rejected": -366.610588592233,
|
|
"loss": 0.5808,
|
|
"loss/base_kto": 3.8420791625976562,
|
|
"metrics/advantage_var": 180.85565185546875,
|
|
"regularization/mmd": 0.8045382499694824,
|
|
"regularization/rkhs_norm": 155.0170135498047,
|
|
"rewards/chosen": 0.17945571392324397,
|
|
"rewards/margins": 0.14586820879253787,
|
|
"rewards/rejected": 0.033587505130706095,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 19.002017974853516,
|
|
"kl": 6.73016357421875,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -37215318.4,
|
|
"logits/rejected": -39168566.4,
|
|
"logps/chosen": -497.32568359375,
|
|
"logps/rejected": -366.280859375,
|
|
"loss": 0.5809,
|
|
"loss/base_kto": 3.782417058944702,
|
|
"metrics/advantage_var": 233.2930145263672,
|
|
"regularization/mmd": 0.8651229739189148,
|
|
"regularization/rkhs_norm": 166.6903839111328,
|
|
"rewards/chosen": 0.1608290195465088,
|
|
"rewards/margins": 0.21101535558700563,
|
|
"rewards/rejected": -0.050186336040496826,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 14.996161460876465,
|
|
"kl": 3.5195395946502686,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36459494.87116564,
|
|
"logits/rejected": -37679361.63057325,
|
|
"logps/chosen": -480.55756326687117,
|
|
"logps/rejected": -366.87684116242036,
|
|
"loss": 0.5869,
|
|
"loss/base_kto": 3.823992967605591,
|
|
"metrics/advantage_var": 237.22227478027344,
|
|
"regularization/mmd": 0.8708967566490173,
|
|
"regularization/rkhs_norm": 167.80284118652344,
|
|
"rewards/chosen": 0.022071779871279475,
|
|
"rewards/margins": 0.1670943968955007,
|
|
"rewards/rejected": -0.14502261702422123,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.5976505279541,
|
|
"kl": 7.171630382537842,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36922570.14886732,
|
|
"logits/rejected": -36722533.31722055,
|
|
"logps/chosen": -470.70782766990294,
|
|
"logps/rejected": -370.5274735649547,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.8298091888427734,
|
|
"metrics/advantage_var": 229.524658203125,
|
|
"regularization/mmd": 0.8734391331672668,
|
|
"regularization/rkhs_norm": 168.29270935058594,
|
|
"rewards/chosen": 0.04818938542338251,
|
|
"rewards/margins": 0.13901804761573583,
|
|
"rewards/rejected": -0.0908286621923533,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 19.52937889099121,
|
|
"kl": 8.829500198364258,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36781159.40065146,
|
|
"logits/rejected": -38217122.21021021,
|
|
"logps/chosen": -473.58978013029315,
|
|
"logps/rejected": -368.9995777027027,
|
|
"loss": 0.5761,
|
|
"loss/base_kto": 3.7992119789123535,
|
|
"metrics/advantage_var": 187.0572967529297,
|
|
"regularization/mmd": 0.8099180459976196,
|
|
"regularization/rkhs_norm": 156.05357360839844,
|
|
"rewards/chosen": 0.10293391473130217,
|
|
"rewards/margins": 0.18581428426709773,
|
|
"rewards/rejected": -0.08288036953579556,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.939891815185547,
|
|
"kl": 11.755973815917969,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36893413.98527246,
|
|
"logits/rejected": -37459828.28618968,
|
|
"logps/chosen": -494.10985824742266,
|
|
"logps/rejected": -379.75785149750413,
|
|
"loss": 0.5873,
|
|
"loss/base_kto": 3.761073350906372,
|
|
"metrics/advantage_var": 249.3211669921875,
|
|
"regularization/mmd": 0.937001645565033,
|
|
"regularization/rkhs_norm": 180.53981018066406,
|
|
"rewards/chosen": 0.21907580548428296,
|
|
"rewards/margins": 0.23387546304252735,
|
|
"rewards/rejected": -0.014799657558244398,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 16.166297912597656,
|
|
"kl": 9.989176750183105,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36024099.64556962,
|
|
"logits/rejected": -37408407.7037037,
|
|
"logps/chosen": -497.938241693038,
|
|
"logps/rejected": -377.37839988425924,
|
|
"loss": 0.5825,
|
|
"loss/base_kto": 3.7838656902313232,
|
|
"metrics/advantage_var": 214.7779541015625,
|
|
"regularization/mmd": 0.8760630488395691,
|
|
"regularization/rkhs_norm": 168.7982940673828,
|
|
"rewards/chosen": 0.17970881884611106,
|
|
"rewards/margins": 0.2030058019998428,
|
|
"rewards/rejected": -0.023296983153731736,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 14.191425323486328,
|
|
"kl": 8.431416511535645,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37001493.590361446,
|
|
"logits/rejected": -37271352.51948052,
|
|
"logps/chosen": -480.8796121987952,
|
|
"logps/rejected": -363.8283532873377,
|
|
"loss": 0.5749,
|
|
"loss/base_kto": 3.765348196029663,
|
|
"metrics/advantage_var": 211.6740264892578,
|
|
"regularization/mmd": 0.8341987729072571,
|
|
"regularization/rkhs_norm": 160.73193359375,
|
|
"rewards/chosen": 0.1759354924581137,
|
|
"rewards/margins": 0.22819954501017592,
|
|
"rewards/rejected": -0.05226405255206219,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 13.746420860290527,
|
|
"kl": 15.271760940551758,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36533016.26911315,
|
|
"logits/rejected": -38252331.34824281,
|
|
"logps/chosen": -528.0307721712538,
|
|
"logps/rejected": -367.827401158147,
|
|
"loss": 0.582,
|
|
"loss/base_kto": 3.7322685718536377,
|
|
"metrics/advantage_var": 246.843017578125,
|
|
"regularization/mmd": 0.9235823750495911,
|
|
"regularization/rkhs_norm": 177.9542236328125,
|
|
"rewards/chosen": 0.3099120720446292,
|
|
"rewards/margins": 0.25135674933050306,
|
|
"rewards/rejected": 0.05855532271412615,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 15.375065803527832,
|
|
"kl": 11.676490783691406,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35755008.0,
|
|
"logits/rejected": -37413446.8427673,
|
|
"logps/chosen": -479.1872573757764,
|
|
"logps/rejected": -356.44545990566036,
|
|
"loss": 0.5768,
|
|
"loss/base_kto": 3.7204177379608154,
|
|
"metrics/advantage_var": 224.97654724121094,
|
|
"regularization/mmd": 0.8937506079673767,
|
|
"regularization/rkhs_norm": 172.20628356933594,
|
|
"rewards/chosen": 0.20321129271702737,
|
|
"rewards/margins": 0.27011555264464504,
|
|
"rewards/rejected": -0.06690425992761768,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 14.875100135803223,
|
|
"kl": 9.409852027893066,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36254610.57735247,
|
|
"logits/rejected": -37408372.826952524,
|
|
"logps/chosen": -465.60446570972886,
|
|
"logps/rejected": -383.4940179938744,
|
|
"loss": 0.5866,
|
|
"loss/base_kto": 3.802095890045166,
|
|
"metrics/advantage_var": 233.0547332763672,
|
|
"regularization/mmd": 0.8909937143325806,
|
|
"regularization/rkhs_norm": 171.6750946044922,
|
|
"rewards/chosen": 0.13589775657349606,
|
|
"rewards/margins": 0.17839135769631861,
|
|
"rewards/rejected": -0.04249360112282255,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 16.19732666015625,
|
|
"kl": 7.737161159515381,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35662791.97568389,
|
|
"logits/rejected": -37826270.25080386,
|
|
"logps/chosen": -469.9730718085106,
|
|
"logps/rejected": -369.18127009646304,
|
|
"loss": 0.5836,
|
|
"loss/base_kto": 3.751713275909424,
|
|
"metrics/advantage_var": 243.52500915527344,
|
|
"regularization/mmd": 0.9167265295982361,
|
|
"regularization/rkhs_norm": 176.6332244873047,
|
|
"rewards/chosen": 0.1811559149559508,
|
|
"rewards/margins": 0.24126886507550904,
|
|
"rewards/rejected": -0.06011295011955826,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 16.072669982910156,
|
|
"kl": 4.069596290588379,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35844065.533884294,
|
|
"logits/rejected": -37826377.95555556,
|
|
"logps/chosen": -483.37551652892563,
|
|
"logps/rejected": -374.82537037037036,
|
|
"loss": 0.588,
|
|
"loss/base_kto": 3.8197503089904785,
|
|
"metrics/advantage_var": 224.0832061767578,
|
|
"regularization/mmd": 0.8839559555053711,
|
|
"regularization/rkhs_norm": 170.31907653808594,
|
|
"rewards/chosen": -0.026373045109520273,
|
|
"rewards/margins": 0.1539941717777772,
|
|
"rewards/rejected": -0.18036721688729745,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 26.276304244995117,
|
|
"kl": 8.102313995361328,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35853812.69400631,
|
|
"logits/rejected": -38136578.37770898,
|
|
"logps/chosen": -493.3289136435331,
|
|
"logps/rejected": -358.9156346749226,
|
|
"loss": 0.5815,
|
|
"loss/base_kto": 3.759909152984619,
|
|
"metrics/advantage_var": 227.8834991455078,
|
|
"regularization/mmd": 0.8921623229980469,
|
|
"regularization/rkhs_norm": 171.9002685546875,
|
|
"rewards/chosen": 0.14323780288455615,
|
|
"rewards/margins": 0.21879939718330566,
|
|
"rewards/rejected": -0.07556159429874952,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 14.230986595153809,
|
|
"kl": 5.5082831382751465,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -34743946.76635514,
|
|
"logits/rejected": -35828774.520376176,
|
|
"logps/chosen": -509.6111273364486,
|
|
"logps/rejected": -383.9224382836991,
|
|
"loss": 0.5802,
|
|
"loss/base_kto": 3.7516236305236816,
|
|
"metrics/advantage_var": 232.79629516601562,
|
|
"regularization/mmd": 0.8902682662010193,
|
|
"regularization/rkhs_norm": 171.53530883789062,
|
|
"rewards/chosen": 0.08423961508682584,
|
|
"rewards/margins": 0.2475465897788566,
|
|
"rewards/rejected": -0.16330697469203076,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 15.873769760131836,
|
|
"kl": 5.017286777496338,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36137547.51377634,
|
|
"logits/rejected": -37503840.91704374,
|
|
"logps/chosen": -467.0659440842788,
|
|
"logps/rejected": -379.62787518853696,
|
|
"loss": 0.5814,
|
|
"loss/base_kto": 3.7632617950439453,
|
|
"metrics/advantage_var": 223.14453125,
|
|
"regularization/mmd": 0.8877536654472351,
|
|
"regularization/rkhs_norm": 171.05081176757812,
|
|
"rewards/chosen": -0.0022734026854776294,
|
|
"rewards/margins": 0.20780201025463832,
|
|
"rewards/rejected": -0.21007541294011595,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 18.570606231689453,
|
|
"kl": 12.657322883605957,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36978017.74545454,
|
|
"logits/rejected": -36951938.477419354,
|
|
"logps/chosen": -480.5526515151515,
|
|
"logps/rejected": -399.71837197580646,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.738701581954956,
|
|
"metrics/advantage_var": 277.3458557128906,
|
|
"regularization/mmd": 0.9612261056900024,
|
|
"regularization/rkhs_norm": 185.20736694335938,
|
|
"rewards/chosen": 0.20457483927408854,
|
|
"rewards/margins": 0.23839894776703208,
|
|
"rewards/rejected": -0.033824108492943546,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 14.51011848449707,
|
|
"kl": 8.948091506958008,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36601050.34049587,
|
|
"logits/rejected": -37118546.67851852,
|
|
"logps/chosen": -481.9414256198347,
|
|
"logps/rejected": -382.19108796296297,
|
|
"loss": 0.5805,
|
|
"loss/base_kto": 3.703148603439331,
|
|
"metrics/advantage_var": 253.4379425048828,
|
|
"regularization/mmd": 0.9405032992362976,
|
|
"regularization/rkhs_norm": 181.2145233154297,
|
|
"rewards/chosen": 0.07828964990032605,
|
|
"rewards/margins": 0.2648555181367265,
|
|
"rewards/rejected": -0.18656586823640045,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.5328369140625,
|
|
"kl": 5.371921062469482,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36305027.2,
|
|
"logits/rejected": -37241644.8,
|
|
"logps/chosen": -511.562548828125,
|
|
"logps/rejected": -377.4674072265625,
|
|
"loss": 0.5831,
|
|
"loss/base_kto": 3.748835802078247,
|
|
"metrics/advantage_var": 242.6811981201172,
|
|
"regularization/mmd": 0.9163103103637695,
|
|
"regularization/rkhs_norm": 176.5530548095703,
|
|
"rewards/chosen": 0.004451106488704682,
|
|
"rewards/margins": 0.25853893607854844,
|
|
"rewards/rejected": -0.25408782958984377,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 12.541788101196289,
|
|
"kl": 6.16095495223999,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36770714.9044586,
|
|
"logits/rejected": -36736420.90797546,
|
|
"logps/chosen": -515.9625298566879,
|
|
"logps/rejected": -396.75814800613495,
|
|
"loss": 0.5791,
|
|
"loss/base_kto": 3.714130401611328,
|
|
"metrics/advantage_var": 238.0179901123047,
|
|
"regularization/mmd": 0.9190575480461121,
|
|
"regularization/rkhs_norm": 177.08238220214844,
|
|
"rewards/chosen": 0.12903496566092132,
|
|
"rewards/margins": 0.27121121347414023,
|
|
"rewards/rejected": -0.14217624781321894,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 15.442559242248535,
|
|
"kl": 10.805193901062012,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36801916.073619634,
|
|
"logits/rejected": -38172711.133757964,
|
|
"logps/chosen": -489.9572469325153,
|
|
"logps/rejected": -368.1966560509554,
|
|
"loss": 0.5832,
|
|
"loss/base_kto": 3.6828064918518066,
|
|
"metrics/advantage_var": 274.185546875,
|
|
"regularization/mmd": 0.9831003546714783,
|
|
"regularization/rkhs_norm": 189.4220428466797,
|
|
"rewards/chosen": 0.22673027647053537,
|
|
"rewards/margins": 0.3061918722852342,
|
|
"rewards/rejected": -0.07946159581469882,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 15.420928955078125,
|
|
"kl": 12.713113784790039,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -35016836.90855457,
|
|
"logits/rejected": -37631452.27906977,
|
|
"logps/chosen": -511.1647308259587,
|
|
"logps/rejected": -350.0207641196013,
|
|
"loss": 0.5726,
|
|
"loss/base_kto": 3.677356719970703,
|
|
"metrics/advantage_var": 239.6293182373047,
|
|
"regularization/mmd": 0.9037864804267883,
|
|
"regularization/rkhs_norm": 174.13998413085938,
|
|
"rewards/chosen": 0.2839052093415837,
|
|
"rewards/margins": 0.2938258396857084,
|
|
"rewards/rejected": -0.009920630344124727,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 24.917781829833984,
|
|
"kl": 14.695927619934082,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35245575.279620856,
|
|
"logits/rejected": -37850765.65069552,
|
|
"logps/chosen": -485.2938882306477,
|
|
"logps/rejected": -346.77511591962906,
|
|
"loss": 0.5758,
|
|
"loss/base_kto": 3.7213807106018066,
|
|
"metrics/advantage_var": 225.18418884277344,
|
|
"regularization/mmd": 0.8851318359375,
|
|
"regularization/rkhs_norm": 170.54563903808594,
|
|
"rewards/chosen": 0.2113020717626876,
|
|
"rewards/margins": 0.22439706961089173,
|
|
"rewards/rejected": -0.013094997848204152,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.418622970581055,
|
|
"kl": 11.514437675476074,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37152716.47798742,
|
|
"logits/rejected": -38735763.875776395,
|
|
"logps/chosen": -480.27181603773585,
|
|
"logps/rejected": -387.8624320652174,
|
|
"loss": 0.5809,
|
|
"loss/base_kto": 3.7262704372406006,
|
|
"metrics/advantage_var": 246.5542449951172,
|
|
"regularization/mmd": 0.9205549359321594,
|
|
"regularization/rkhs_norm": 177.3708953857422,
|
|
"rewards/chosen": 0.19024605421150256,
|
|
"rewards/margins": 0.2572571292017218,
|
|
"rewards/rejected": -0.06701107499021922,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 16.732126235961914,
|
|
"kl": 8.530130386352539,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37478884.884735204,
|
|
"logits/rejected": -38562087.322884016,
|
|
"logps/chosen": -481.5382593457944,
|
|
"logps/rejected": -366.82631269592474,
|
|
"loss": 0.5829,
|
|
"loss/base_kto": 3.753072738647461,
|
|
"metrics/advantage_var": 234.08889770507812,
|
|
"regularization/mmd": 0.9098120927810669,
|
|
"regularization/rkhs_norm": 175.3009796142578,
|
|
"rewards/chosen": 0.09287618476653768,
|
|
"rewards/margins": 0.23414263986379596,
|
|
"rewards/rejected": -0.14126645509725827,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 16.767581939697266,
|
|
"kl": 8.381802558898926,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37358098.8416,
|
|
"logits/rejected": -37499430.302290075,
|
|
"logps/chosen": -495.5541,
|
|
"logps/rejected": -365.99797232824426,
|
|
"loss": 0.5782,
|
|
"loss/base_kto": 3.7136998176574707,
|
|
"metrics/advantage_var": 236.6925811767578,
|
|
"regularization/mmd": 0.9121416211128235,
|
|
"regularization/rkhs_norm": 175.7498321533203,
|
|
"rewards/chosen": 0.154965087890625,
|
|
"rewards/margins": 0.2942994389424797,
|
|
"rewards/rejected": -0.1393343510518547,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 14.336687088012695,
|
|
"kl": 8.144498825073242,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37520804.14352574,
|
|
"logits/rejected": -37166802.729264475,
|
|
"logps/chosen": -463.71548361934475,
|
|
"logps/rejected": -387.0881748826291,
|
|
"loss": 0.5821,
|
|
"loss/base_kto": 3.768022298812866,
|
|
"metrics/advantage_var": 229.356689453125,
|
|
"regularization/mmd": 0.8890665173530579,
|
|
"regularization/rkhs_norm": 171.3037567138672,
|
|
"rewards/chosen": 0.07597560555254242,
|
|
"rewards/margins": 0.20010504268423013,
|
|
"rewards/rejected": -0.1241294371316877,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 15.663066864013672,
|
|
"kl": 13.2122220993042,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36791159.254658386,
|
|
"logits/rejected": -37065747.320754714,
|
|
"logps/chosen": -476.13955745341616,
|
|
"logps/rejected": -381.0774862421384,
|
|
"loss": 0.5793,
|
|
"loss/base_kto": 3.674278974533081,
|
|
"metrics/advantage_var": 260.3456726074219,
|
|
"regularization/mmd": 0.9603710174560547,
|
|
"regularization/rkhs_norm": 185.04258728027344,
|
|
"rewards/chosen": 0.20968898038686432,
|
|
"rewards/margins": 0.29388162826084474,
|
|
"rewards/rejected": -0.08419264787398044,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 14.14006233215332,
|
|
"kl": 10.773856163024902,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -38245590.01929261,
|
|
"logits/rejected": -39144485.34954407,
|
|
"logps/chosen": -485.81064107717043,
|
|
"logps/rejected": -367.3224021656535,
|
|
"loss": 0.5822,
|
|
"loss/base_kto": 3.6606903076171875,
|
|
"metrics/advantage_var": 294.3084411621094,
|
|
"regularization/mmd": 0.9966379404067993,
|
|
"regularization/rkhs_norm": 192.0304412841797,
|
|
"rewards/chosen": 0.18594723422427653,
|
|
"rewards/margins": 0.30308339234132153,
|
|
"rewards/rejected": -0.11713615811704502,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 13.075891494750977,
|
|
"kl": 9.43057918548584,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -38252587.908116385,
|
|
"logits/rejected": -38680824.24242424,
|
|
"logps/chosen": -481.020769525268,
|
|
"logps/rejected": -371.8990231259968,
|
|
"loss": 0.5783,
|
|
"loss/base_kto": 3.6810624599456787,
|
|
"metrics/advantage_var": 252.88096618652344,
|
|
"regularization/mmd": 0.9453756213188171,
|
|
"regularization/rkhs_norm": 182.15330505371094,
|
|
"rewards/chosen": 0.14692261536672688,
|
|
"rewards/margins": 0.2867086253301564,
|
|
"rewards/rejected": -0.13978600996342952,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 11.875099182128906,
|
|
"kl": 16.19339370727539,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36562400.58895706,
|
|
"logits/rejected": -36876072.76433121,
|
|
"logps/chosen": -467.93989647239266,
|
|
"logps/rejected": -381.3198895302548,
|
|
"loss": 0.5776,
|
|
"loss/base_kto": 3.723682403564453,
|
|
"metrics/advantage_var": 240.4628448486328,
|
|
"regularization/mmd": 0.8968575596809387,
|
|
"regularization/rkhs_norm": 172.804931640625,
|
|
"rewards/chosen": 0.2558060160443827,
|
|
"rewards/margins": 0.2370044757182217,
|
|
"rewards/rejected": 0.01880154032616099,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.578527450561523,
|
|
"kl": 17.684675216674805,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36594226.06940063,
|
|
"logits/rejected": -38848794.15479876,
|
|
"logps/chosen": -499.7094834384858,
|
|
"logps/rejected": -345.8370501160991,
|
|
"loss": 0.5752,
|
|
"loss/base_kto": 3.688239336013794,
|
|
"metrics/advantage_var": 236.94468688964844,
|
|
"regularization/mmd": 0.9133060574531555,
|
|
"regularization/rkhs_norm": 175.9741973876953,
|
|
"rewards/chosen": 0.3275871878542735,
|
|
"rewards/margins": 0.2631883071407782,
|
|
"rewards/rejected": 0.06439888071349531,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 17.35607147216797,
|
|
"kl": 10.405311584472656,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35766881.37423313,
|
|
"logits/rejected": -36893578.598726116,
|
|
"logps/chosen": -467.71875,
|
|
"logps/rejected": -373.93033439490443,
|
|
"loss": 0.5863,
|
|
"loss/base_kto": 3.7130889892578125,
|
|
"metrics/advantage_var": 268.3462219238281,
|
|
"regularization/mmd": 0.9771196246147156,
|
|
"regularization/rkhs_norm": 188.26968383789062,
|
|
"rewards/chosen": 0.1244574236723543,
|
|
"rewards/margins": 0.25826325140237516,
|
|
"rewards/rejected": -0.1338058277300209,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 14.678869247436523,
|
|
"kl": 13.108316421508789,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -35862636.83091787,
|
|
"logits/rejected": -37128653.345509894,
|
|
"logps/chosen": -441.8135064412238,
|
|
"logps/rejected": -349.9011843607306,
|
|
"loss": 0.5816,
|
|
"loss/base_kto": 3.532607316970825,
|
|
"metrics/advantage_var": 760.6506958007812,
|
|
"regularization/mmd": 1.12008798122406,
|
|
"regularization/rkhs_norm": 215.81655883789062,
|
|
"rewards/chosen": 0.28411821006000904,
|
|
"rewards/margins": 0.4328765510668643,
|
|
"rewards/rejected": -0.14875834100685525,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 8.693588256835938,
|
|
"kl": 7.911380290985107,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -37739820.78972713,
|
|
"logits/rejected": -37636853.47945205,
|
|
"logps/chosen": -479.6870485553772,
|
|
"logps/rejected": -375.3836092085236,
|
|
"loss": 0.5817,
|
|
"loss/base_kto": 3.2553489208221436,
|
|
"metrics/advantage_var": 602.6400756835938,
|
|
"regularization/mmd": 1.3985315561294556,
|
|
"regularization/rkhs_norm": 269.46661376953125,
|
|
"rewards/chosen": 0.34640764998777335,
|
|
"rewards/margins": 0.8268036594590918,
|
|
"rewards/rejected": -0.4803960094713185,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 14.268240928649902,
|
|
"kl": 13.279520034790039,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35715484.17610063,
|
|
"logits/rejected": -37000484.571428575,
|
|
"logps/chosen": -480.6579205974843,
|
|
"logps/rejected": -373.34452639751555,
|
|
"loss": 0.5553,
|
|
"loss/base_kto": 3.225963592529297,
|
|
"metrics/advantage_var": 433.2372131347656,
|
|
"regularization/mmd": 1.2162116765975952,
|
|
"regularization/rkhs_norm": 234.33750915527344,
|
|
"rewards/chosen": 0.4841145449464426,
|
|
"rewards/margins": 0.8107024351923422,
|
|
"rewards/rejected": -0.3265878902458996,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 22.305408477783203,
|
|
"kl": 10.857390403747559,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35868313.3640553,
|
|
"logits/rejected": -37256301.07472178,
|
|
"logps/chosen": -484.29896313364054,
|
|
"logps/rejected": -371.7512668918919,
|
|
"loss": 0.5744,
|
|
"loss/base_kto": 3.207843542098999,
|
|
"metrics/advantage_var": 607.4768676757812,
|
|
"regularization/mmd": 1.3877257108688354,
|
|
"regularization/rkhs_norm": 267.384521484375,
|
|
"rewards/chosen": 0.4639082526281682,
|
|
"rewards/margins": 0.8864520193958549,
|
|
"rewards/rejected": -0.4225437667676868,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 17.167024612426758,
|
|
"kl": 10.920756340026855,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36573758.205607474,
|
|
"logits/rejected": -36726231.87460815,
|
|
"logps/chosen": -478.85669781931466,
|
|
"logps/rejected": -389.09149686520374,
|
|
"loss": 0.5621,
|
|
"loss/base_kto": 3.2149360179901123,
|
|
"metrics/advantage_var": 522.9557495117188,
|
|
"regularization/mmd": 1.2821253538131714,
|
|
"regularization/rkhs_norm": 247.0376434326172,
|
|
"rewards/chosen": 0.44286063601294784,
|
|
"rewards/margins": 0.8642979144495906,
|
|
"rewards/rejected": -0.4214372784366428,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 15.37991714477539,
|
|
"kl": 17.988571166992188,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34962014.105919,
|
|
"logits/rejected": -37623618.60815047,
|
|
"logps/chosen": -463.96631619937693,
|
|
"logps/rejected": -377.11265673981194,
|
|
"loss": 0.5744,
|
|
"loss/base_kto": 3.18288516998291,
|
|
"metrics/advantage_var": 666.353515625,
|
|
"regularization/mmd": 1.412236213684082,
|
|
"regularization/rkhs_norm": 272.107177734375,
|
|
"rewards/chosen": 0.6200395625699717,
|
|
"rewards/margins": 0.8824504110619633,
|
|
"rewards/rejected": -0.2624108484919916,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 13.74785327911377,
|
|
"kl": 11.522727012634277,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34761661.38577912,
|
|
"logits/rejected": -35597482.39095315,
|
|
"logps/chosen": -484.8576966717095,
|
|
"logps/rejected": -368.3999394184168,
|
|
"loss": 0.5724,
|
|
"loss/base_kto": 3.2111737728118896,
|
|
"metrics/advantage_var": 623.5446166992188,
|
|
"regularization/mmd": 1.3679980039596558,
|
|
"regularization/rkhs_norm": 263.5834655761719,
|
|
"rewards/chosen": 0.5148262797253215,
|
|
"rewards/margins": 0.8707250366985142,
|
|
"rewards/rejected": -0.3558987569731927,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 8.256586074829102,
|
|
"kl": 11.941576957702637,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35060964.41881639,
|
|
"logits/rejected": -36416876.41867955,
|
|
"logps/chosen": -483.2103566009105,
|
|
"logps/rejected": -373.48998590982285,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.177649736404419,
|
|
"metrics/advantage_var": 542.2218627929688,
|
|
"regularization/mmd": 1.320379614830017,
|
|
"regularization/rkhs_norm": 254.40843200683594,
|
|
"rewards/chosen": 0.5500806886617982,
|
|
"rewards/margins": 0.8763316587374699,
|
|
"rewards/rejected": -0.3262509700756718,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 13.353851318359375,
|
|
"kl": 11.167734146118164,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35917364.448780484,
|
|
"logits/rejected": -37759576.54135338,
|
|
"logps/chosen": -504.52423780487806,
|
|
"logps/rejected": -371.3499530075188,
|
|
"loss": 0.5703,
|
|
"loss/base_kto": 3.2346246242523193,
|
|
"metrics/advantage_var": 558.4301147460938,
|
|
"regularization/mmd": 1.3278635740280151,
|
|
"regularization/rkhs_norm": 255.8503875732422,
|
|
"rewards/chosen": 0.4285968563420986,
|
|
"rewards/margins": 0.8302321396186494,
|
|
"rewards/rejected": -0.40163528327655074,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 10.750252723693848,
|
|
"kl": 10.83349895477295,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35720616.96784074,
|
|
"logits/rejected": -36187741.90749601,
|
|
"logps/chosen": -466.7466979326187,
|
|
"logps/rejected": -363.6123903508772,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.2108311653137207,
|
|
"metrics/advantage_var": 598.1678466796875,
|
|
"regularization/mmd": 1.3379846811294556,
|
|
"regularization/rkhs_norm": 257.8005065917969,
|
|
"rewards/chosen": 0.46493291927880936,
|
|
"rewards/margins": 0.878182687987691,
|
|
"rewards/rejected": -0.4132497687088816,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 12.432683944702148,
|
|
"kl": 16.59678840637207,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35164084.61349693,
|
|
"logits/rejected": -36045840.30573248,
|
|
"logps/chosen": -495.03613880368096,
|
|
"logps/rejected": -385.9206807324841,
|
|
"loss": 0.5645,
|
|
"loss/base_kto": 3.131800413131714,
|
|
"metrics/advantage_var": 611.6915283203125,
|
|
"regularization/mmd": 1.3845771551132202,
|
|
"regularization/rkhs_norm": 266.77789306640625,
|
|
"rewards/chosen": 0.56035684222824,
|
|
"rewards/margins": 0.9517924046535983,
|
|
"rewards/rejected": -0.39143556242535826,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 12.830517768859863,
|
|
"kl": 14.84776496887207,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34960491.70486656,
|
|
"logits/rejected": -36466044.615863144,
|
|
"logps/chosen": -490.01785714285717,
|
|
"logps/rejected": -392.803824844479,
|
|
"loss": 0.5606,
|
|
"loss/base_kto": 3.130321979522705,
|
|
"metrics/advantage_var": 579.6293334960938,
|
|
"regularization/mmd": 1.3548129796981812,
|
|
"regularization/rkhs_norm": 261.04296875,
|
|
"rewards/chosen": 0.5789561129439756,
|
|
"rewards/margins": 0.932766234945944,
|
|
"rewards/rejected": -0.3538101220019683,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 10.16787338256836,
|
|
"kl": 13.066168785095215,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -36429967.262321144,
|
|
"logits/rejected": -36910075.28110599,
|
|
"logps/chosen": -502.24155405405406,
|
|
"logps/rejected": -371.7343029953917,
|
|
"loss": 0.5736,
|
|
"loss/base_kto": 3.1768033504486084,
|
|
"metrics/advantage_var": 652.4033203125,
|
|
"regularization/mmd": 1.4117525815963745,
|
|
"regularization/rkhs_norm": 272.0140075683594,
|
|
"rewards/chosen": 0.5260245561220688,
|
|
"rewards/margins": 0.9127794724992169,
|
|
"rewards/rejected": -0.38675491637714815,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 12.876511573791504,
|
|
"kl": 10.898401260375977,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35050169.741324924,
|
|
"logits/rejected": -37410181.94427244,
|
|
"logps/chosen": -473.98501577287067,
|
|
"logps/rejected": -385.85032894736844,
|
|
"loss": 0.5742,
|
|
"loss/base_kto": 3.2272300720214844,
|
|
"metrics/advantage_var": 680.0474243164062,
|
|
"regularization/mmd": 1.366205096244812,
|
|
"regularization/rkhs_norm": 263.2380065917969,
|
|
"rewards/chosen": 0.4491842853534109,
|
|
"rewards/margins": 0.8463826583035367,
|
|
"rewards/rejected": -0.3971983729501258,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 15.82406234741211,
|
|
"kl": 13.82843017578125,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34766848.0,
|
|
"logits/rejected": -36481363.2392638,
|
|
"logps/chosen": -472.88808718152865,
|
|
"logps/rejected": -365.3694401840491,
|
|
"loss": 0.5662,
|
|
"loss/base_kto": 3.1631321907043457,
|
|
"metrics/advantage_var": 557.621337890625,
|
|
"regularization/mmd": 1.366207242012024,
|
|
"regularization/rkhs_norm": 263.2383728027344,
|
|
"rewards/chosen": 0.4635902939328722,
|
|
"rewards/margins": 0.9067686229325846,
|
|
"rewards/rejected": -0.44317832899971243,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 12.57764720916748,
|
|
"kl": 12.005444526672363,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34430386.470764615,
|
|
"logits/rejected": -36824536.743882544,
|
|
"logps/chosen": -458.97732383808096,
|
|
"logps/rejected": -376.22394983686786,
|
|
"loss": 0.5582,
|
|
"loss/base_kto": 3.203517198562622,
|
|
"metrics/advantage_var": 494.190673828125,
|
|
"regularization/mmd": 1.261722445487976,
|
|
"regularization/rkhs_norm": 243.1064453125,
|
|
"rewards/chosen": 0.5004961966038465,
|
|
"rewards/margins": 0.8768474333174986,
|
|
"rewards/rejected": -0.3763512367136521,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 14.65658950805664,
|
|
"kl": 19.227481842041016,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34804148.822256565,
|
|
"logits/rejected": -36089092.448657185,
|
|
"logps/chosen": -500.9465803709428,
|
|
"logps/rejected": -395.6623222748815,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.1164557933807373,
|
|
"metrics/advantage_var": 609.3770751953125,
|
|
"regularization/mmd": 1.4019100666046143,
|
|
"regularization/rkhs_norm": 270.1175231933594,
|
|
"rewards/chosen": 0.6837412909341191,
|
|
"rewards/margins": 0.9595717451051055,
|
|
"rewards/rejected": -0.2758304541709864,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 17.153152465820312,
|
|
"kl": 14.933130264282227,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34955676.8,
|
|
"logits/rejected": -33910611.2,
|
|
"logps/chosen": -507.87060546875,
|
|
"logps/rejected": -383.27626953125,
|
|
"loss": 0.5644,
|
|
"loss/base_kto": 3.164644956588745,
|
|
"metrics/advantage_var": 554.138671875,
|
|
"regularization/mmd": 1.3509505987167358,
|
|
"regularization/rkhs_norm": 260.2987365722656,
|
|
"rewards/chosen": 0.5595901489257813,
|
|
"rewards/margins": 0.9042657852172852,
|
|
"rewards/rejected": -0.3446756362915039,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.022993087768555,
|
|
"kl": 14.624844551086426,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34782373.05790298,
|
|
"logits/rejected": -35916258.44617785,
|
|
"logps/chosen": -465.64940336463224,
|
|
"logps/rejected": -375.3988884555382,
|
|
"loss": 0.5774,
|
|
"loss/base_kto": 3.235236406326294,
|
|
"metrics/advantage_var": 589.6281127929688,
|
|
"regularization/mmd": 1.3840060234069824,
|
|
"regularization/rkhs_norm": 266.6678161621094,
|
|
"rewards/chosen": 0.5337352484044894,
|
|
"rewards/margins": 0.8444082405097204,
|
|
"rewards/rejected": -0.3106729921052311,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 11.805953979492188,
|
|
"kl": 12.613221168518066,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34975839.403726704,
|
|
"logits/rejected": -35924727.94968554,
|
|
"logps/chosen": -463.35956909937886,
|
|
"logps/rejected": -384.19256092767296,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.1837172508239746,
|
|
"metrics/advantage_var": 537.3102416992188,
|
|
"regularization/mmd": 1.2904276847839355,
|
|
"regularization/rkhs_norm": 248.63731384277344,
|
|
"rewards/chosen": 0.4899637447380871,
|
|
"rewards/margins": 0.8880418474736905,
|
|
"rewards/rejected": -0.3980781027356034,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 16.55732536315918,
|
|
"kl": 10.782447814941406,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35219584.20031299,
|
|
"logits/rejected": -36587542.3650546,
|
|
"logps/chosen": -491.5154538341158,
|
|
"logps/rejected": -377.14632897815915,
|
|
"loss": 0.5705,
|
|
"loss/base_kto": 3.250697374343872,
|
|
"metrics/advantage_var": 539.7772827148438,
|
|
"regularization/mmd": 1.3133488893508911,
|
|
"regularization/rkhs_norm": 253.0537109375,
|
|
"rewards/chosen": 0.44366784610658744,
|
|
"rewards/margins": 0.8023400372325138,
|
|
"rewards/rejected": -0.3586721911259263,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 11.22537612915039,
|
|
"kl": 11.976910591125488,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35918954.797546014,
|
|
"logits/rejected": -36634930.5477707,
|
|
"logps/chosen": -486.192053297546,
|
|
"logps/rejected": -389.9213525079618,
|
|
"loss": 0.5639,
|
|
"loss/base_kto": 3.148801565170288,
|
|
"metrics/advantage_var": 602.7506103515625,
|
|
"regularization/mmd": 1.3627887964248657,
|
|
"regularization/rkhs_norm": 262.5797424316406,
|
|
"rewards/chosen": 0.6037709990893405,
|
|
"rewards/margins": 0.9375360168478497,
|
|
"rewards/rejected": -0.33376501775850914,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 12.934151649475098,
|
|
"kl": 10.646471977233887,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34417081.35193799,
|
|
"logits/rejected": -36333474.46929134,
|
|
"logps/chosen": -478.5356589147287,
|
|
"logps/rejected": -371.8240649606299,
|
|
"loss": 0.5595,
|
|
"loss/base_kto": 3.2100167274475098,
|
|
"metrics/advantage_var": 512.4363403320312,
|
|
"regularization/mmd": 1.2661036252975464,
|
|
"regularization/rkhs_norm": 243.9506072998047,
|
|
"rewards/chosen": 0.4641662124515504,
|
|
"rewards/margins": 0.8582134344388166,
|
|
"rewards/rejected": -0.3940472219872662,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 14.983908653259277,
|
|
"kl": 13.93272590637207,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35439073.88235294,
|
|
"logits/rejected": -36827613.97730956,
|
|
"logps/chosen": -489.1387160633484,
|
|
"logps/rejected": -363.0306928687196,
|
|
"loss": 0.5678,
|
|
"loss/base_kto": 3.194981336593628,
|
|
"metrics/advantage_var": 636.0108032226562,
|
|
"regularization/mmd": 1.3470886945724487,
|
|
"regularization/rkhs_norm": 259.5546569824219,
|
|
"rewards/chosen": 0.6047796879418835,
|
|
"rewards/margins": 0.9126137381992402,
|
|
"rewards/rejected": -0.30783405025735666,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 11.32305908203125,
|
|
"kl": 15.7722749710083,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34840466.343307085,
|
|
"logits/rejected": -35493929.27751938,
|
|
"logps/chosen": -465.68641732283464,
|
|
"logps/rejected": -377.9899224806202,
|
|
"loss": 0.5717,
|
|
"loss/base_kto": 3.2660393714904785,
|
|
"metrics/advantage_var": 634.408203125,
|
|
"regularization/mmd": 1.3071792125701904,
|
|
"regularization/rkhs_norm": 251.864990234375,
|
|
"rewards/chosen": 0.5427279735174705,
|
|
"rewards/margins": 0.798872250217749,
|
|
"rewards/rejected": -0.25614427670027856,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 12.453219413757324,
|
|
"kl": 14.714609146118164,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35550287.01732284,
|
|
"logits/rejected": -37225257.674418606,
|
|
"logps/chosen": -463.03208661417324,
|
|
"logps/rejected": -385.7218023255814,
|
|
"loss": 0.5603,
|
|
"loss/base_kto": 3.1603755950927734,
|
|
"metrics/advantage_var": 547.91650390625,
|
|
"regularization/mmd": 1.3222483396530151,
|
|
"regularization/rkhs_norm": 254.76846313476562,
|
|
"rewards/chosen": 0.5865217554287647,
|
|
"rewards/margins": 0.897254385485693,
|
|
"rewards/rejected": -0.3107326300569283,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 10.28625202178955,
|
|
"kl": 11.269328117370605,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34310377.235569425,
|
|
"logits/rejected": -37046586.090766825,
|
|
"logps/chosen": -486.41936427457097,
|
|
"logps/rejected": -378.6247310250391,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.199897527694702,
|
|
"metrics/advantage_var": 531.6617431640625,
|
|
"regularization/mmd": 1.2984915971755981,
|
|
"regularization/rkhs_norm": 250.19107055664062,
|
|
"rewards/chosen": 0.47318538005192085,
|
|
"rewards/margins": 0.8691493801986345,
|
|
"rewards/rejected": -0.3959640001467136,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 13.333651542663574,
|
|
"kl": 13.84549617767334,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34690342.14439324,
|
|
"logits/rejected": -37146295.14785374,
|
|
"logps/chosen": -500.7231182795699,
|
|
"logps/rejected": -350.54640302066775,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.2646405696868896,
|
|
"metrics/advantage_var": 498.6590270996094,
|
|
"regularization/mmd": 1.2358344793319702,
|
|
"regularization/rkhs_norm": 238.118408203125,
|
|
"rewards/chosen": 0.48771637009768626,
|
|
"rewards/margins": 0.8031107852935577,
|
|
"rewards/rejected": -0.3153944151958714,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 10.942756652832031,
|
|
"kl": 10.66589641571045,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35559077.90387597,
|
|
"logits/rejected": -37027675.51496063,
|
|
"logps/chosen": -496.4138565891473,
|
|
"logps/rejected": -395.9872539370079,
|
|
"loss": 0.5716,
|
|
"loss/base_kto": 3.1901156902313232,
|
|
"metrics/advantage_var": 618.5126953125,
|
|
"regularization/mmd": 1.3826411962509155,
|
|
"regularization/rkhs_norm": 266.4048767089844,
|
|
"rewards/chosen": 0.5244257490764292,
|
|
"rewards/margins": 0.8879523892400611,
|
|
"rewards/rejected": -0.3635266401636319,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 11.067506790161133,
|
|
"kl": 11.680200576782227,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34297272.6885759,
|
|
"logits/rejected": -35808877.429017164,
|
|
"logps/chosen": -481.43412558685446,
|
|
"logps/rejected": -364.6852574102964,
|
|
"loss": 0.5613,
|
|
"loss/base_kto": 3.2230727672576904,
|
|
"metrics/advantage_var": 551.4659423828125,
|
|
"regularization/mmd": 1.2670122385025024,
|
|
"regularization/rkhs_norm": 244.1256866455078,
|
|
"rewards/chosen": 0.516129280293305,
|
|
"rewards/margins": 0.8489735547576429,
|
|
"rewards/rejected": -0.33284427446433795,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 11.165182113647461,
|
|
"kl": 7.130992889404297,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35079116.40916031,
|
|
"logits/rejected": -37472718.0288,
|
|
"logps/chosen": -504.9458492366412,
|
|
"logps/rejected": -386.8584,
|
|
"loss": 0.5727,
|
|
"loss/base_kto": 3.242945909500122,
|
|
"metrics/advantage_var": 574.9050903320312,
|
|
"regularization/mmd": 1.3384863138198853,
|
|
"regularization/rkhs_norm": 257.8971862792969,
|
|
"rewards/chosen": 0.44571374791269086,
|
|
"rewards/margins": 0.8575791287720659,
|
|
"rewards/rejected": -0.411865380859375,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 13.952322006225586,
|
|
"kl": 13.516879081726074,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -36339054.29570747,
|
|
"logits/rejected": -37394991.97542243,
|
|
"logps/chosen": -472.8339626391097,
|
|
"logps/rejected": -383.9543010752688,
|
|
"loss": 0.5666,
|
|
"loss/base_kto": 3.2231850624084473,
|
|
"metrics/advantage_var": 580.9198608398438,
|
|
"regularization/mmd": 1.3093918561935425,
|
|
"regularization/rkhs_norm": 252.2913055419922,
|
|
"rewards/chosen": 0.4768674331931886,
|
|
"rewards/margins": 0.8558001223852969,
|
|
"rewards/rejected": -0.3789326891921083,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 10.066413879394531,
|
|
"kl": 13.076878547668457,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36395267.6056338,
|
|
"logits/rejected": -36595494.73946958,
|
|
"logps/chosen": -477.75762910798124,
|
|
"logps/rejected": -376.4611446957878,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.211498260498047,
|
|
"metrics/advantage_var": 533.5299682617188,
|
|
"regularization/mmd": 1.288972020149231,
|
|
"regularization/rkhs_norm": 248.3568572998047,
|
|
"rewards/chosen": 0.48883429155663144,
|
|
"rewards/margins": 0.844412185937055,
|
|
"rewards/rejected": -0.35557789438042364,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 11.708639144897461,
|
|
"kl": 11.991992950439453,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35184009.6,
|
|
"logits/rejected": -36124099.2,
|
|
"logps/chosen": -514.879296875,
|
|
"logps/rejected": -360.0527587890625,
|
|
"loss": 0.5627,
|
|
"loss/base_kto": 3.2546794414520264,
|
|
"metrics/advantage_var": 495.984130859375,
|
|
"regularization/mmd": 1.246625304222107,
|
|
"regularization/rkhs_norm": 240.1975555419922,
|
|
"rewards/chosen": 0.44855966567993166,
|
|
"rewards/margins": 0.8007103443145752,
|
|
"rewards/rejected": -0.35215067863464355,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 10.563225746154785,
|
|
"kl": 8.732428550720215,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34164239.81788618,
|
|
"logits/rejected": -35446112.62556391,
|
|
"logps/chosen": -446.88805894308945,
|
|
"logps/rejected": -390.9661889097744,
|
|
"loss": 0.5618,
|
|
"loss/base_kto": 3.2683663368225098,
|
|
"metrics/advantage_var": 487.888916015625,
|
|
"regularization/mmd": 1.2264251708984375,
|
|
"regularization/rkhs_norm": 236.305419921875,
|
|
"rewards/chosen": 0.3575970657472688,
|
|
"rewards/margins": 0.7771152367852152,
|
|
"rewards/rejected": -0.4195181710379464,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 12.504398345947266,
|
|
"kl": 9.695332527160645,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35459299.55555555,
|
|
"logits/rejected": -35925389.06122449,
|
|
"logps/chosen": -473.90482954545456,
|
|
"logps/rejected": -368.60784894314867,
|
|
"loss": 0.5617,
|
|
"loss/base_kto": 3.1839234828948975,
|
|
"metrics/advantage_var": 539.5236206054688,
|
|
"regularization/mmd": 1.309561848640442,
|
|
"regularization/rkhs_norm": 252.32408142089844,
|
|
"rewards/chosen": 0.355194091796875,
|
|
"rewards/margins": 0.8811144147600446,
|
|
"rewards/rejected": -0.5259203229631696,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 9.233758926391602,
|
|
"kl": 9.138747215270996,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35134499.73899848,
|
|
"logits/rejected": -34734772.56038647,
|
|
"logps/chosen": -452.3097022003035,
|
|
"logps/rejected": -363.14042371175526,
|
|
"loss": 0.5661,
|
|
"loss/base_kto": 3.228759765625,
|
|
"metrics/advantage_var": 568.5081176757812,
|
|
"regularization/mmd": 1.3004276752471924,
|
|
"regularization/rkhs_norm": 250.5640869140625,
|
|
"rewards/chosen": 0.4284956936409332,
|
|
"rewards/margins": 0.8713627085519889,
|
|
"rewards/rejected": -0.44286701491105573,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 11.373794555664062,
|
|
"kl": 15.229090690612793,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34968732.31152648,
|
|
"logits/rejected": -35761681.655172415,
|
|
"logps/chosen": -464.3160046728972,
|
|
"logps/rejected": -364.5926234326019,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.2107696533203125,
|
|
"metrics/advantage_var": 511.4872131347656,
|
|
"regularization/mmd": 1.2473591566085815,
|
|
"regularization/rkhs_norm": 240.33897399902344,
|
|
"rewards/chosen": 0.524776755835037,
|
|
"rewards/margins": 0.854089956304338,
|
|
"rewards/rejected": -0.329313200469301,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 10.809823036193848,
|
|
"kl": 9.994755744934082,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35793925.06095552,
|
|
"logits/rejected": -36806674.258543834,
|
|
"logps/chosen": -501.436676276771,
|
|
"logps/rejected": -375.8259658246657,
|
|
"loss": 0.5583,
|
|
"loss/base_kto": 3.1938979625701904,
|
|
"metrics/advantage_var": 484.18829345703125,
|
|
"regularization/mmd": 1.2723420858383179,
|
|
"regularization/rkhs_norm": 245.1526336669922,
|
|
"rewards/chosen": 0.425955607431528,
|
|
"rewards/margins": 0.8633833265704117,
|
|
"rewards/rejected": -0.43742771913888373,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 10.175517082214355,
|
|
"kl": 7.224442481994629,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34654372.102564104,
|
|
"logits/rejected": -36665635.2293578,
|
|
"logps/chosen": -493.4987980769231,
|
|
"logps/rejected": -376.8448012232416,
|
|
"loss": 0.5391,
|
|
"loss/base_kto": 3.090583562850952,
|
|
"metrics/advantage_var": 437.85943603515625,
|
|
"regularization/mmd": 1.2219974994659424,
|
|
"regularization/rkhs_norm": 235.4523162841797,
|
|
"rewards/chosen": 0.443420899220002,
|
|
"rewards/margins": 0.9589343747372516,
|
|
"rewards/rejected": -0.5155134755172496,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 7.5480146408081055,
|
|
"kl": 9.547552108764648,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34524432.32348367,
|
|
"logits/rejected": -35836174.06593407,
|
|
"logps/chosen": -473.12830482115083,
|
|
"logps/rejected": -364.38255494505495,
|
|
"loss": 0.5258,
|
|
"loss/base_kto": 3.0715863704681396,
|
|
"metrics/advantage_var": 376.4017028808594,
|
|
"regularization/mmd": 1.1350246667861938,
|
|
"regularization/rkhs_norm": 218.6945343017578,
|
|
"rewards/chosen": 0.5007050367387976,
|
|
"rewards/margins": 0.9446626425049622,
|
|
"rewards/rejected": -0.44395760576616466,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 9.177377700805664,
|
|
"kl": 15.20007038116455,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -36705515.016393445,
|
|
"logits/rejected": -36971394.674626864,
|
|
"logps/chosen": -486.7358094262295,
|
|
"logps/rejected": -377.20275186567164,
|
|
"loss": 0.5285,
|
|
"loss/base_kto": 3.0651772022247314,
|
|
"metrics/advantage_var": 394.803466796875,
|
|
"regularization/mmd": 1.1631301641464233,
|
|
"regularization/rkhs_norm": 224.10986328125,
|
|
"rewards/chosen": 0.6062375928534836,
|
|
"rewards/margins": 0.9591697855278072,
|
|
"rewards/rejected": -0.3529321926743237,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 10.426277160644531,
|
|
"kl": 7.440224647521973,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35711407.655384615,
|
|
"logits/rejected": -36049276.342857145,
|
|
"logps/chosen": -465.85201923076926,
|
|
"logps/rejected": -376.7583829365079,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.116962432861328,
|
|
"metrics/advantage_var": 364.39337158203125,
|
|
"regularization/mmd": 1.1236332654953003,
|
|
"regularization/rkhs_norm": 216.49966430664062,
|
|
"rewards/chosen": 0.4489228703425481,
|
|
"rewards/margins": 0.9276707680204994,
|
|
"rewards/rejected": -0.4787478976779514,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 9.994585037231445,
|
|
"kl": 11.306876182556152,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -32935320.96594427,
|
|
"logits/rejected": -33967058.776025236,
|
|
"logps/chosen": -480.172116873065,
|
|
"logps/rejected": -357.9880717665615,
|
|
"loss": 0.5262,
|
|
"loss/base_kto": 3.1046035289764404,
|
|
"metrics/advantage_var": 352.54736328125,
|
|
"regularization/mmd": 1.1049422025680542,
|
|
"regularization/rkhs_norm": 212.89830017089844,
|
|
"rewards/chosen": 0.5738749312173471,
|
|
"rewards/margins": 0.9094383261429189,
|
|
"rewards/rejected": -0.33556339492557175,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 10.421794891357422,
|
|
"kl": 13.09758186340332,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35574125.598736174,
|
|
"logits/rejected": -37259403.276661515,
|
|
"logps/chosen": -472.6114731437599,
|
|
"logps/rejected": -361.852033423493,
|
|
"loss": 0.533,
|
|
"loss/base_kto": 3.0979297161102295,
|
|
"metrics/advantage_var": 437.20343017578125,
|
|
"regularization/mmd": 1.16596257686615,
|
|
"regularization/rkhs_norm": 224.6555938720703,
|
|
"rewards/chosen": 0.5889343936673085,
|
|
"rewards/margins": 0.9401041272483798,
|
|
"rewards/rejected": -0.3511697335810713,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 13.22502326965332,
|
|
"kl": 11.006478309631348,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34004045.34138973,
|
|
"logits/rejected": -35296537.682847895,
|
|
"logps/chosen": -448.5730740181269,
|
|
"logps/rejected": -382.09921116504853,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.1165502071380615,
|
|
"metrics/advantage_var": 330.66534423828125,
|
|
"regularization/mmd": 1.0836093425750732,
|
|
"regularization/rkhs_norm": 208.7879180908203,
|
|
"rewards/chosen": 0.544666209609847,
|
|
"rewards/margins": 0.899138891538068,
|
|
"rewards/rejected": -0.35447268192822107,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 12.145218849182129,
|
|
"kl": 13.656302452087402,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34741664.83563748,
|
|
"logits/rejected": -36443951.61844197,
|
|
"logps/chosen": -483.568356374808,
|
|
"logps/rejected": -386.0348022655008,
|
|
"loss": 0.5313,
|
|
"loss/base_kto": 3.0872457027435303,
|
|
"metrics/advantage_var": 445.0165100097656,
|
|
"regularization/mmd": 1.162818193435669,
|
|
"regularization/rkhs_norm": 224.0497283935547,
|
|
"rewards/chosen": 0.5681736443632392,
|
|
"rewards/margins": 0.9339168478220404,
|
|
"rewards/rejected": -0.3657432034588012,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 8.214476585388184,
|
|
"kl": 13.03216552734375,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34539516.7232,
|
|
"logits/rejected": -36572095.902290076,
|
|
"logps/chosen": -469.64255,
|
|
"logps/rejected": -379.549856870229,
|
|
"loss": 0.5293,
|
|
"loss/base_kto": 3.066572904586792,
|
|
"metrics/advantage_var": 386.48052978515625,
|
|
"regularization/mmd": 1.1677439212799072,
|
|
"regularization/rkhs_norm": 224.9988250732422,
|
|
"rewards/chosen": 0.5685177734375,
|
|
"rewards/margins": 0.9507694954675572,
|
|
"rewards/rejected": -0.38225172203005725,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 12.507845878601074,
|
|
"kl": 11.591044425964355,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34798801.84899846,
|
|
"logits/rejected": -36189341.41362916,
|
|
"logps/chosen": -508.97033898305085,
|
|
"logps/rejected": -371.386588748019,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.116471767425537,
|
|
"metrics/advantage_var": 328.8663635253906,
|
|
"regularization/mmd": 1.0820915699005127,
|
|
"regularization/rkhs_norm": 208.4954833984375,
|
|
"rewards/chosen": 0.4973133716083638,
|
|
"rewards/margins": 0.9042345648717849,
|
|
"rewards/rejected": -0.40692119326342113,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 13.732808113098145,
|
|
"kl": 12.445700645446777,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35190004.90529695,
|
|
"logits/rejected": -36013266.4109589,
|
|
"logps/chosen": -448.63242375601925,
|
|
"logps/rejected": -390.4064640410959,
|
|
"loss": 0.5222,
|
|
"loss/base_kto": 3.0124518871307373,
|
|
"metrics/advantage_var": 396.3167419433594,
|
|
"regularization/mmd": 1.164847731590271,
|
|
"regularization/rkhs_norm": 224.44082641601562,
|
|
"rewards/chosen": 0.5731517124329103,
|
|
"rewards/margins": 1.0015430282356834,
|
|
"rewards/rejected": -0.428391315802773,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 11.632913589477539,
|
|
"kl": 9.960004806518555,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -35090586.32339089,
|
|
"logits/rejected": -35364316.16796268,
|
|
"logps/chosen": -476.7870879120879,
|
|
"logps/rejected": -381.368050155521,
|
|
"loss": 0.534,
|
|
"loss/base_kto": 3.0374245643615723,
|
|
"metrics/advantage_var": 500.48095703125,
|
|
"regularization/mmd": 1.2344404458999634,
|
|
"regularization/rkhs_norm": 237.8498077392578,
|
|
"rewards/chosen": 0.5685330904447116,
|
|
"rewards/margins": 1.042508288339103,
|
|
"rewards/rejected": -0.4739751978943915,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 8.724961280822754,
|
|
"kl": 11.3777437210083,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35454945.38714499,
|
|
"logits/rejected": -36630293.368248776,
|
|
"logps/chosen": -480.68735986547085,
|
|
"logps/rejected": -384.0438829787234,
|
|
"loss": 0.5246,
|
|
"loss/base_kto": 3.077766180038452,
|
|
"metrics/advantage_var": 358.9744567871094,
|
|
"regularization/mmd": 1.1187477111816406,
|
|
"regularization/rkhs_norm": 215.55833435058594,
|
|
"rewards/chosen": 0.5505154870550729,
|
|
"rewards/margins": 0.9326538033078859,
|
|
"rewards/rejected": -0.382138316252813,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 12.175980567932129,
|
|
"kl": 16.86451530456543,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34500447.69329073,
|
|
"logits/rejected": -36395922.39755352,
|
|
"logps/chosen": -477.5335463258786,
|
|
"logps/rejected": -344.38704128440367,
|
|
"loss": 0.5303,
|
|
"loss/base_kto": 3.070604085922241,
|
|
"metrics/advantage_var": 409.4195556640625,
|
|
"regularization/mmd": 1.1718789339065552,
|
|
"regularization/rkhs_norm": 225.79556274414062,
|
|
"rewards/chosen": 0.5908082224690495,
|
|
"rewards/margins": 0.9275099218221888,
|
|
"rewards/rejected": -0.3367016993531393,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 12.980863571166992,
|
|
"kl": 14.155290603637695,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34901440.20447284,
|
|
"logits/rejected": -36133145.83486239,
|
|
"logps/chosen": -502.09285143769966,
|
|
"logps/rejected": -350.33010798929666,
|
|
"loss": 0.5311,
|
|
"loss/base_kto": 3.090366840362549,
|
|
"metrics/advantage_var": 388.5970764160156,
|
|
"regularization/mmd": 1.15811288356781,
|
|
"regularization/rkhs_norm": 223.1431427001953,
|
|
"rewards/chosen": 0.559625497641274,
|
|
"rewards/margins": 0.907635966197156,
|
|
"rewards/rejected": -0.3480104685558821,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 9.398697853088379,
|
|
"kl": 8.864479064941406,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33870864.05015674,
|
|
"logits/rejected": -35462421.532710284,
|
|
"logps/chosen": -484.74549373040753,
|
|
"logps/rejected": -372.36180880062307,
|
|
"loss": 0.5257,
|
|
"loss/base_kto": 3.0728273391723633,
|
|
"metrics/advantage_var": 369.09527587890625,
|
|
"regularization/mmd": 1.1326578855514526,
|
|
"regularization/rkhs_norm": 218.2384796142578,
|
|
"rewards/chosen": 0.5317208699671826,
|
|
"rewards/margins": 0.964871544662349,
|
|
"rewards/rejected": -0.4331506746951665,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 11.3624906539917,
|
|
"kl": 9.979337692260742,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35131661.38827258,
|
|
"logits/rejected": -36449988.4375963,
|
|
"logps/chosen": -488.05432844690966,
|
|
"logps/rejected": -404.49239214175657,
|
|
"loss": 0.5328,
|
|
"loss/base_kto": 3.107598066329956,
|
|
"metrics/advantage_var": 379.69091796875,
|
|
"regularization/mmd": 1.1545418500900269,
|
|
"regularization/rkhs_norm": 222.45506286621094,
|
|
"rewards/chosen": 0.4826808633214887,
|
|
"rewards/margins": 0.9221636594952898,
|
|
"rewards/rejected": -0.43948279617380104,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 13.29701042175293,
|
|
"kl": 12.003888130187988,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34585973.110754415,
|
|
"logits/rejected": -35529988.28614917,
|
|
"logps/chosen": -489.06154695024077,
|
|
"logps/rejected": -379.20362442922374,
|
|
"loss": 0.5259,
|
|
"loss/base_kto": 3.0957882404327393,
|
|
"metrics/advantage_var": 377.1058044433594,
|
|
"regularization/mmd": 1.1110972166061401,
|
|
"regularization/rkhs_norm": 214.084228515625,
|
|
"rewards/chosen": 0.5009159192227628,
|
|
"rewards/margins": 0.9204459429456509,
|
|
"rewards/rejected": -0.4195300237228881,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 10.090277671813965,
|
|
"kl": 9.778497695922852,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34456244.01826484,
|
|
"logits/rejected": -35405646.4847512,
|
|
"logps/chosen": -475.02254566210047,
|
|
"logps/rejected": -393.3794642857143,
|
|
"loss": 0.5314,
|
|
"loss/base_kto": 3.087869644165039,
|
|
"metrics/advantage_var": 398.113525390625,
|
|
"regularization/mmd": 1.1632604598999023,
|
|
"regularization/rkhs_norm": 224.1349639892578,
|
|
"rewards/chosen": 0.5226121783437976,
|
|
"rewards/margins": 0.9577280843994256,
|
|
"rewards/rejected": -0.435115906055628,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 10.796884536743164,
|
|
"kl": 11.28317928314209,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -35882955.1483871,
|
|
"logits/rejected": -36496371.587878786,
|
|
"logps/chosen": -484.7116935483871,
|
|
"logps/rejected": -373.4014204545455,
|
|
"loss": 0.5346,
|
|
"loss/base_kto": 3.0655720233917236,
|
|
"metrics/advantage_var": 690.8641967773438,
|
|
"regularization/mmd": 1.211446762084961,
|
|
"regularization/rkhs_norm": 233.4193878173828,
|
|
"rewards/chosen": 0.5636926958637852,
|
|
"rewards/margins": 0.9260875026967634,
|
|
"rewards/rejected": -0.3623948068329782,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 11.23148250579834,
|
|
"kl": 8.112841606140137,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35362990.03338391,
|
|
"logits/rejected": -36551088.0257649,
|
|
"logps/chosen": -492.5554817905918,
|
|
"logps/rejected": -383.8329559178744,
|
|
"loss": 0.534,
|
|
"loss/base_kto": 3.1174371242523193,
|
|
"metrics/advantage_var": 378.5379333496094,
|
|
"regularization/mmd": 1.1546032428741455,
|
|
"regularization/rkhs_norm": 222.46690368652344,
|
|
"rewards/chosen": 0.5165647932177068,
|
|
"rewards/margins": 0.9326679903984133,
|
|
"rewards/rejected": -0.41610319718070654,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 10.351691246032715,
|
|
"kl": 12.127572059631348,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34929444.222575516,
|
|
"logits/rejected": -36351981.124423966,
|
|
"logps/chosen": -483.57740461049286,
|
|
"logps/rejected": -351.6439132104455,
|
|
"loss": 0.5294,
|
|
"loss/base_kto": 3.141176700592041,
|
|
"metrics/advantage_var": 342.2101745605469,
|
|
"regularization/mmd": 1.0943201780319214,
|
|
"regularization/rkhs_norm": 210.8516845703125,
|
|
"rewards/chosen": 0.518753124534231,
|
|
"rewards/margins": 0.8818327991334053,
|
|
"rewards/rejected": -0.36307967459917434,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 8.668177604675293,
|
|
"kl": 11.561732292175293,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34681797.573573574,
|
|
"logits/rejected": -37524756.84690554,
|
|
"logps/chosen": -484.3523836336336,
|
|
"logps/rejected": -366.404417752443,
|
|
"loss": 0.5277,
|
|
"loss/base_kto": 3.0814101696014404,
|
|
"metrics/advantage_var": 396.5794677734375,
|
|
"regularization/mmd": 1.140345811843872,
|
|
"regularization/rkhs_norm": 219.71983337402344,
|
|
"rewards/chosen": 0.5470803750527872,
|
|
"rewards/margins": 0.9517047672857752,
|
|
"rewards/rejected": -0.4046243922329881,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 9.765106201171875,
|
|
"kl": 12.24731159210205,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35110081.3869969,
|
|
"logits/rejected": -35376900.03785489,
|
|
"logps/chosen": -475.60110294117646,
|
|
"logps/rejected": -401.7883231466877,
|
|
"loss": 0.5286,
|
|
"loss/base_kto": 3.0967602729797363,
|
|
"metrics/advantage_var": 378.4054870605469,
|
|
"regularization/mmd": 1.1319931745529175,
|
|
"regularization/rkhs_norm": 218.1104278564453,
|
|
"rewards/chosen": 0.5609224396224362,
|
|
"rewards/margins": 0.9206717372367894,
|
|
"rewards/rejected": -0.3597492976143533,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 11.077553749084473,
|
|
"kl": 12.09903621673584,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34868436.18018018,
|
|
"logits/rejected": -35288560.99022801,
|
|
"logps/chosen": -477.72142454954957,
|
|
"logps/rejected": -377.02020561889253,
|
|
"loss": 0.5294,
|
|
"loss/base_kto": 3.1114776134490967,
|
|
"metrics/advantage_var": 379.4000549316406,
|
|
"regularization/mmd": 1.1235958337783813,
|
|
"regularization/rkhs_norm": 216.492431640625,
|
|
"rewards/chosen": 0.5875523655980199,
|
|
"rewards/margins": 0.9047007295377338,
|
|
"rewards/rejected": -0.31714836393971396,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 11.563310623168945,
|
|
"kl": 14.744864463806152,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34062047.6416,
|
|
"logits/rejected": -36122086.20458015,
|
|
"logps/chosen": -501.57185,
|
|
"logps/rejected": -373.68578244274806,
|
|
"loss": 0.5231,
|
|
"loss/base_kto": 3.0718495845794678,
|
|
"metrics/advantage_var": 351.4624328613281,
|
|
"regularization/mmd": 1.1132713556289673,
|
|
"regularization/rkhs_norm": 214.50315856933594,
|
|
"rewards/chosen": 0.56427158203125,
|
|
"rewards/margins": 0.9246223524719706,
|
|
"rewards/rejected": -0.36035077044072045,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 13.124002456665039,
|
|
"kl": 12.960229873657227,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35166417.45454545,
|
|
"logits/rejected": -36333947.37349398,
|
|
"logps/chosen": -505.40584415584414,
|
|
"logps/rejected": -371.461125753012,
|
|
"loss": 0.5214,
|
|
"loss/base_kto": 3.0765461921691895,
|
|
"metrics/advantage_var": 338.88116455078125,
|
|
"regularization/mmd": 1.0944137573242188,
|
|
"regularization/rkhs_norm": 210.8696746826172,
|
|
"rewards/chosen": 0.5693161704323508,
|
|
"rewards/margins": 0.9247379950599628,
|
|
"rewards/rejected": -0.355421824627612,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 10.356775283813477,
|
|
"kl": 11.589325904846191,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35005978.52433281,
|
|
"logits/rejected": -36816548.827371694,
|
|
"logps/chosen": -479.6517857142857,
|
|
"logps/rejected": -380.92493681959564,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.1003997325897217,
|
|
"metrics/advantage_var": 368.20941162109375,
|
|
"regularization/mmd": 1.1266430616378784,
|
|
"regularization/rkhs_norm": 217.07958984375,
|
|
"rewards/chosen": 0.5164729766606162,
|
|
"rewards/margins": 0.9221262464743555,
|
|
"rewards/rejected": -0.4056532698137393,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 8.896793365478516,
|
|
"kl": 11.762689590454102,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34398214.61712439,
|
|
"logits/rejected": -34162052.84114977,
|
|
"logps/chosen": -464.71940630048465,
|
|
"logps/rejected": -387.36365355521934,
|
|
"loss": 0.5225,
|
|
"loss/base_kto": 3.0738818645477295,
|
|
"metrics/advantage_var": 352.2379150390625,
|
|
"regularization/mmd": 1.1062079668045044,
|
|
"regularization/rkhs_norm": 213.14219665527344,
|
|
"rewards/chosen": 0.5081033783698505,
|
|
"rewards/margins": 0.9525329000315137,
|
|
"rewards/rejected": -0.4444295216616632,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 11.057719230651855,
|
|
"kl": 10.35720443725586,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -33739849.8018018,
|
|
"logits/rejected": -35829566.54071661,
|
|
"logps/chosen": -486.15019707207205,
|
|
"logps/rejected": -354.3031606270358,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.13413405418396,
|
|
"metrics/advantage_var": 341.60284423828125,
|
|
"regularization/mmd": 1.095909595489502,
|
|
"regularization/rkhs_norm": 211.1579132080078,
|
|
"rewards/chosen": 0.4855748998510229,
|
|
"rewards/margins": 0.8819497545197551,
|
|
"rewards/rejected": -0.3963748546687322,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 10.408794403076172,
|
|
"kl": 9.436334609985352,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34858039.565891474,
|
|
"logits/rejected": -35625743.72283465,
|
|
"logps/chosen": -495.32374031007754,
|
|
"logps/rejected": -365.80300196850396,
|
|
"loss": 0.5305,
|
|
"loss/base_kto": 3.1316869258880615,
|
|
"metrics/advantage_var": 343.2964782714844,
|
|
"regularization/mmd": 1.112682580947876,
|
|
"regularization/rkhs_norm": 214.3896942138672,
|
|
"rewards/chosen": 0.5042510749757751,
|
|
"rewards/margins": 0.8946056940395055,
|
|
"rewards/rejected": -0.3903546190637303,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 10.051740646362305,
|
|
"kl": 11.429483413696289,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35000138.24405705,
|
|
"logits/rejected": -35360932.092449926,
|
|
"logps/chosen": -476.41615491283676,
|
|
"logps/rejected": -382.5160342835131,
|
|
"loss": 0.5263,
|
|
"loss/base_kto": 3.0970723628997803,
|
|
"metrics/advantage_var": 365.07769775390625,
|
|
"regularization/mmd": 1.1132915019989014,
|
|
"regularization/rkhs_norm": 214.5070343017578,
|
|
"rewards/chosen": 0.547252914984895,
|
|
"rewards/margins": 0.9334721453663714,
|
|
"rewards/rejected": -0.3862192303814763,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 10.373788833618164,
|
|
"kl": 11.397631645202637,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34736305.53560372,
|
|
"logits/rejected": -35973627.15457413,
|
|
"logps/chosen": -491.89599458204333,
|
|
"logps/rejected": -372.0601094242902,
|
|
"loss": 0.523,
|
|
"loss/base_kto": 3.1050448417663574,
|
|
"metrics/advantage_var": 324.3363952636719,
|
|
"regularization/mmd": 1.0786465406417847,
|
|
"regularization/rkhs_norm": 207.83169555664062,
|
|
"rewards/chosen": 0.49661018678647445,
|
|
"rewards/margins": 0.8968850952915021,
|
|
"rewards/rejected": -0.4002749085050276,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 10.832305908203125,
|
|
"kl": 13.42828369140625,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35967585.37423313,
|
|
"logits/rejected": -37460085.401273884,
|
|
"logps/chosen": -497.1944018404908,
|
|
"logps/rejected": -398.7984175955414,
|
|
"loss": 0.5375,
|
|
"loss/base_kto": 3.0646770000457764,
|
|
"metrics/advantage_var": 447.9042053222656,
|
|
"regularization/mmd": 1.2349318265914917,
|
|
"regularization/rkhs_norm": 237.94448852539062,
|
|
"rewards/chosen": 0.5716980074080953,
|
|
"rewards/margins": 0.9821908740699932,
|
|
"rewards/rejected": -0.4104928666618979,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 9.45889949798584,
|
|
"kl": 11.008593559265137,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34225293.50371471,
|
|
"logits/rejected": -35967131.202635914,
|
|
"logps/chosen": -452.90629643387814,
|
|
"logps/rejected": -373.5236563014827,
|
|
"loss": 0.5268,
|
|
"loss/base_kto": 3.1598641872406006,
|
|
"metrics/advantage_var": 315.8893127441406,
|
|
"regularization/mmd": 1.0546658039093018,
|
|
"regularization/rkhs_norm": 203.2111358642578,
|
|
"rewards/chosen": 0.4887530254573737,
|
|
"rewards/margins": 0.8587430647611989,
|
|
"rewards/rejected": -0.3699900393038252,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 11.392977714538574,
|
|
"kl": 12.263094902038574,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33919995.19248826,
|
|
"logits/rejected": -35588246.16536661,
|
|
"logps/chosen": -486.4821987480438,
|
|
"logps/rejected": -370.3834584633385,
|
|
"loss": 0.524,
|
|
"loss/base_kto": 3.0500006675720215,
|
|
"metrics/advantage_var": 388.0404968261719,
|
|
"regularization/mmd": 1.1416443586349487,
|
|
"regularization/rkhs_norm": 219.97000122070312,
|
|
"rewards/chosen": 0.5924555580007824,
|
|
"rewards/margins": 0.982419264459197,
|
|
"rewards/rejected": -0.3899637064584146,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 8.935293197631836,
|
|
"kl": 13.46716022491455,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34426445.17341977,
|
|
"logits/rejected": -34715193.91855203,
|
|
"logps/chosen": -496.82151539708263,
|
|
"logps/rejected": -380.5034407993967,
|
|
"loss": 0.5276,
|
|
"loss/base_kto": 3.088454008102417,
|
|
"metrics/advantage_var": 354.8756408691406,
|
|
"regularization/mmd": 1.1322225332260132,
|
|
"regularization/rkhs_norm": 218.15464782714844,
|
|
"rewards/chosen": 0.5244452230740985,
|
|
"rewards/margins": 0.910459237961153,
|
|
"rewards/rejected": -0.38601401488705456,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 7.952939510345459,
|
|
"kl": 10.338269233703613,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -35913489.27607362,
|
|
"logits/rejected": -36257058.24203822,
|
|
"logps/chosen": -468.9126725460123,
|
|
"logps/rejected": -391.99477507961785,
|
|
"loss": 0.5307,
|
|
"loss/base_kto": 3.1191604137420654,
|
|
"metrics/advantage_var": 356.20672607421875,
|
|
"regularization/mmd": 1.1262457370758057,
|
|
"regularization/rkhs_norm": 217.0030059814453,
|
|
"rewards/chosen": 0.5046517775833972,
|
|
"rewards/margins": 0.9003884719193347,
|
|
"rewards/rejected": -0.3957366943359375,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_loss": 0.5588559634944935,
|
|
"train_runtime": 11100.6813,
|
|
"train_samples_per_second": 13.352,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|