5fd1fe8a03
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed0 Source: Original Platform
2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 15.571517944335938,
|
|
"kl": 9.538911819458008,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36670855.86141732,
|
|
"logits/rejected": -37428016.0248062,
|
|
"logps/chosen": -466.5294291338583,
|
|
"logps/rejected": -375.1725048449612,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.9124398231506348,
|
|
"metrics/advantage_var": 246.2978973388672,
|
|
"regularization/mmd": 0.8353201150894165,
|
|
"regularization/rkhs_norm": 160.94802856445312,
|
|
"rewards/chosen": 0.10693647730068898,
|
|
"rewards/margins": 0.0829938729400346,
|
|
"rewards/rejected": 0.023942604360654374,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 15.307286262512207,
|
|
"kl": 13.87574291229248,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36265901.95377504,
|
|
"logits/rejected": -37981487.467511885,
|
|
"logps/chosen": -484.0126155624037,
|
|
"logps/rejected": -376.0209488906498,
|
|
"loss": 0.5871,
|
|
"loss/base_kto": 3.873422622680664,
|
|
"metrics/advantage_var": 193.25048828125,
|
|
"regularization/mmd": 0.8232390284538269,
|
|
"regularization/rkhs_norm": 158.6202392578125,
|
|
"rewards/chosen": 0.183217758390312,
|
|
"rewards/margins": 0.11884899200464566,
|
|
"rewards/rejected": 0.06436876638566635,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 18.164213180541992,
|
|
"kl": 5.212839603424072,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35386226.70387597,
|
|
"logits/rejected": -35990765.05196851,
|
|
"logps/chosen": -475.18464147286824,
|
|
"logps/rejected": -359.7203248031496,
|
|
"loss": 0.5808,
|
|
"loss/base_kto": 3.8760621547698975,
|
|
"metrics/advantage_var": 169.1094512939453,
|
|
"regularization/mmd": 0.7703275084495544,
|
|
"regularization/rkhs_norm": 148.4253387451172,
|
|
"rewards/chosen": 0.020032202550607135,
|
|
"rewards/margins": 0.1362361594549809,
|
|
"rewards/rejected": -0.11620395690437377,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 16.888856887817383,
|
|
"kl": 10.327300071716309,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37882630.66462481,
|
|
"logits/rejected": -38822433.480063796,
|
|
"logps/chosen": -491.083269525268,
|
|
"logps/rejected": -376.98327850877195,
|
|
"loss": 0.5776,
|
|
"loss/base_kto": 3.8541600704193115,
|
|
"metrics/advantage_var": 165.6694793701172,
|
|
"regularization/mmd": 0.7665037512779236,
|
|
"regularization/rkhs_norm": 147.68858337402344,
|
|
"rewards/chosen": 0.15566759562236793,
|
|
"rewards/margins": 0.13684607704529114,
|
|
"rewards/rejected": 0.01882151857707679,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 18.683528900146484,
|
|
"kl": 3.284613847732544,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36451557.45819398,
|
|
"logits/rejected": -37508225.126099706,
|
|
"logps/chosen": -508.8169941471572,
|
|
"logps/rejected": -383.2355434384164,
|
|
"loss": 0.5815,
|
|
"loss/base_kto": 3.8655879497528076,
|
|
"metrics/advantage_var": 177.95997619628906,
|
|
"regularization/mmd": 0.7861188650131226,
|
|
"regularization/rkhs_norm": 151.4679718017578,
|
|
"rewards/chosen": -0.03227000411936272,
|
|
"rewards/margins": 0.11076771918697435,
|
|
"rewards/rejected": -0.14303772330633707,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 11.06960678100586,
|
|
"kl": 11.392313003540039,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35050959.12363067,
|
|
"logits/rejected": -36804598.4149766,
|
|
"logps/chosen": -455.0399061032864,
|
|
"logps/rejected": -358.3792414196568,
|
|
"loss": 0.5873,
|
|
"loss/base_kto": 3.881218671798706,
|
|
"metrics/advantage_var": 196.70751953125,
|
|
"regularization/mmd": 0.8169800043106079,
|
|
"regularization/rkhs_norm": 157.4142608642578,
|
|
"rewards/chosen": 0.115661728550011,
|
|
"rewards/margins": 0.10624722828225615,
|
|
"rewards/rejected": 0.00941450026775485,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 13.950078010559082,
|
|
"kl": 5.728837490081787,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36749391.50310559,
|
|
"logits/rejected": -37346703.295597486,
|
|
"logps/chosen": -500.00349378881987,
|
|
"logps/rejected": -381.5553508254717,
|
|
"loss": 0.5863,
|
|
"loss/base_kto": 3.8131446838378906,
|
|
"metrics/advantage_var": 231.8813934326172,
|
|
"regularization/mmd": 0.8771948218345642,
|
|
"regularization/rkhs_norm": 169.01634216308594,
|
|
"rewards/chosen": 0.05716702952888442,
|
|
"rewards/margins": 0.1870318241820512,
|
|
"rewards/rejected": -0.12986479465316678,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 13.237503051757812,
|
|
"kl": 8.087309837341309,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35618828.52599388,
|
|
"logits/rejected": -36276878.31309904,
|
|
"logps/chosen": -473.78631498470946,
|
|
"logps/rejected": -364.7073682108626,
|
|
"loss": 0.5756,
|
|
"loss/base_kto": 3.8134605884552,
|
|
"metrics/advantage_var": 181.14297485351562,
|
|
"regularization/mmd": 0.7911037802696228,
|
|
"regularization/rkhs_norm": 152.428466796875,
|
|
"rewards/chosen": 0.08350314569035801,
|
|
"rewards/margins": 0.15822221822118368,
|
|
"rewards/rejected": -0.07471907253082567,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 14.653188705444336,
|
|
"kl": 15.084848403930664,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36715595.38650307,
|
|
"logits/rejected": -37102428.94267516,
|
|
"logps/chosen": -461.5396855828221,
|
|
"logps/rejected": -364.40159733280257,
|
|
"loss": 0.5793,
|
|
"loss/base_kto": 3.7889657020568848,
|
|
"metrics/advantage_var": 204.2191162109375,
|
|
"regularization/mmd": 0.8457900881767273,
|
|
"regularization/rkhs_norm": 162.96536254882812,
|
|
"rewards/chosen": 0.30467050938518503,
|
|
"rewards/margins": 0.18966756745177787,
|
|
"rewards/rejected": 0.11500294193340714,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 18.993202209472656,
|
|
"kl": 4.71709680557251,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -38046149.71076923,
|
|
"logits/rejected": -37486377.44761905,
|
|
"logps/chosen": -493.4542307692308,
|
|
"logps/rejected": -403.08010912698415,
|
|
"loss": 0.5922,
|
|
"loss/base_kto": 3.8570022583007812,
|
|
"metrics/advantage_var": 212.0679931640625,
|
|
"regularization/mmd": 0.8802224397659302,
|
|
"regularization/rkhs_norm": 169.59970092773438,
|
|
"rewards/chosen": 0.022366710075965295,
|
|
"rewards/margins": 0.14284531289404565,
|
|
"rewards/rejected": -0.12047860281808036,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 13.953816413879395,
|
|
"kl": 10.587577819824219,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37216005.22448979,
|
|
"logits/rejected": -37344576.09953344,
|
|
"logps/chosen": -492.0768249607535,
|
|
"logps/rejected": -378.4935361586314,
|
|
"loss": 0.5776,
|
|
"loss/base_kto": 3.748526096343994,
|
|
"metrics/advantage_var": 219.2888641357422,
|
|
"regularization/mmd": 0.8723026514053345,
|
|
"regularization/rkhs_norm": 168.07373046875,
|
|
"rewards/chosen": 0.2412963340196355,
|
|
"rewards/margins": 0.24378986309467665,
|
|
"rewards/rejected": -0.0024935290750411586,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 14.870015144348145,
|
|
"kl": 6.1637091636657715,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -38293815.508090615,
|
|
"logits/rejected": -38198009.03927492,
|
|
"logps/chosen": -481.43669093851133,
|
|
"logps/rejected": -382.0043665030212,
|
|
"loss": 0.5936,
|
|
"loss/base_kto": 3.8108742237091064,
|
|
"metrics/advantage_var": 250.5630340576172,
|
|
"regularization/mmd": 0.937593936920166,
|
|
"regularization/rkhs_norm": 180.6539306640625,
|
|
"rewards/chosen": -0.0679220366246492,
|
|
"rewards/margins": 0.1553817207807039,
|
|
"rewards/rejected": -0.2233037574053531,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 14.09887981414795,
|
|
"kl": 5.046163558959961,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37280157.602605864,
|
|
"logits/rejected": -38798062.318318315,
|
|
"logps/chosen": -517.2627239413681,
|
|
"logps/rejected": -373.02587743993996,
|
|
"loss": 0.5833,
|
|
"loss/base_kto": 3.738848924636841,
|
|
"metrics/advantage_var": 246.8894805908203,
|
|
"regularization/mmd": 0.9277191162109375,
|
|
"regularization/rkhs_norm": 178.7512664794922,
|
|
"rewards/chosen": 0.027881609888729134,
|
|
"rewards/margins": 0.25842825761066984,
|
|
"rewards/rejected": -0.2305466477219407,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 13.609427452087402,
|
|
"kl": 2.7668838500976562,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37892866.048,
|
|
"logits/rejected": -39231050.25954199,
|
|
"logps/chosen": -516.8509,
|
|
"logps/rejected": -391.6226383587786,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.731579542160034,
|
|
"metrics/advantage_var": 260.7747497558594,
|
|
"regularization/mmd": 0.9528587460517883,
|
|
"regularization/rkhs_norm": 183.5951385498047,
|
|
"rewards/chosen": -0.024977178955078123,
|
|
"rewards/margins": 0.26962076709543475,
|
|
"rewards/rejected": -0.29459794605051287,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 16.254486083984375,
|
|
"kl": 10.548303604125977,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36928320.0,
|
|
"logits/rejected": -37830617.6,
|
|
"logps/chosen": -495.545849609375,
|
|
"logps/rejected": -350.2385498046875,
|
|
"loss": 0.577,
|
|
"loss/base_kto": 3.7107200622558594,
|
|
"metrics/advantage_var": 235.25523376464844,
|
|
"regularization/mmd": 0.9048828482627869,
|
|
"regularization/rkhs_norm": 174.35121154785156,
|
|
"rewards/chosen": 0.22926266193389894,
|
|
"rewards/margins": 0.27641730308532714,
|
|
"rewards/rejected": -0.04715464115142822,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 12.882833480834961,
|
|
"kl": 7.569650173187256,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36503129.24770642,
|
|
"logits/rejected": -39226525.03514377,
|
|
"logps/chosen": -489.1180237003058,
|
|
"logps/rejected": -384.365964456869,
|
|
"loss": 0.5802,
|
|
"loss/base_kto": 3.788947343826294,
|
|
"metrics/advantage_var": 206.8526153564453,
|
|
"regularization/mmd": 0.8527728319168091,
|
|
"regularization/rkhs_norm": 164.31076049804688,
|
|
"rewards/chosen": 0.1230209770552609,
|
|
"rewards/margins": 0.2077236120190064,
|
|
"rewards/rejected": -0.08470263496374551,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 20.220701217651367,
|
|
"kl": 12.921002388000488,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -38969773.67228178,
|
|
"logits/rejected": -39175548.52950558,
|
|
"logps/chosen": -491.80560872894335,
|
|
"logps/rejected": -378.8329096889952,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.8205342292785645,
|
|
"metrics/advantage_var": 231.05068969726562,
|
|
"regularization/mmd": 0.9027809500694275,
|
|
"regularization/rkhs_norm": 173.9462127685547,
|
|
"rewards/chosen": 0.20647296058456163,
|
|
"rewards/margins": 0.1769182832327515,
|
|
"rewards/rejected": 0.029554677351810145,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 14.828634262084961,
|
|
"kl": 4.833045959472656,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36490830.76923077,
|
|
"logits/rejected": -37599659.70731708,
|
|
"logps/chosen": -496.5906951121795,
|
|
"logps/rejected": -385.7433307926829,
|
|
"loss": 0.5778,
|
|
"loss/base_kto": 3.7393481731414795,
|
|
"metrics/advantage_var": 224.12744140625,
|
|
"regularization/mmd": 0.8829835057258606,
|
|
"regularization/rkhs_norm": 170.13169860839844,
|
|
"rewards/chosen": 0.005749409015362079,
|
|
"rewards/margins": 0.24529392857936266,
|
|
"rewards/rejected": -0.23954451956400058,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 13.264820098876953,
|
|
"kl": 12.306333541870117,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35814038.49455676,
|
|
"logits/rejected": -37112094.14128728,
|
|
"logps/chosen": -483.37218118195955,
|
|
"logps/rejected": -352.5729984301413,
|
|
"loss": 0.5764,
|
|
"loss/base_kto": 3.719007968902588,
|
|
"metrics/advantage_var": 225.1729278564453,
|
|
"regularization/mmd": 0.8922660946846008,
|
|
"regularization/rkhs_norm": 171.9202423095703,
|
|
"rewards/chosen": 0.22712349187159797,
|
|
"rewards/margins": 0.24754749481393326,
|
|
"rewards/rejected": -0.020424002942335286,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 14.280707359313965,
|
|
"kl": 3.4532058238983154,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35375217.42769231,
|
|
"logits/rejected": -37906880.60952381,
|
|
"logps/chosen": -508.5975480769231,
|
|
"logps/rejected": -366.37363591269843,
|
|
"loss": 0.5886,
|
|
"loss/base_kto": 3.743018388748169,
|
|
"metrics/advantage_var": 267.3111267089844,
|
|
"regularization/mmd": 0.9658026099205017,
|
|
"regularization/rkhs_norm": 186.089111328125,
|
|
"rewards/chosen": 0.09360163175142729,
|
|
"rewards/margins": 0.2721868174533122,
|
|
"rewards/rejected": -0.17858518570188492,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 18.786409378051758,
|
|
"kl": 7.645198822021484,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36261818.469135806,
|
|
"logits/rejected": -36752089.11392405,
|
|
"logps/chosen": -511.5447048611111,
|
|
"logps/rejected": -390.44521360759495,
|
|
"loss": 0.5841,
|
|
"loss/base_kto": 3.760416030883789,
|
|
"metrics/advantage_var": 239.8439483642578,
|
|
"regularization/mmd": 0.9127063751220703,
|
|
"regularization/rkhs_norm": 175.858642578125,
|
|
"rewards/chosen": 0.11989230874144001,
|
|
"rewards/margins": 0.2177080094655951,
|
|
"rewards/rejected": -0.09781570072415509,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 22.86400032043457,
|
|
"kl": 7.945260524749756,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36764127.38853503,
|
|
"logits/rejected": -37778906.306748465,
|
|
"logps/chosen": -489.1418192675159,
|
|
"logps/rejected": -377.61277799079755,
|
|
"loss": 0.5949,
|
|
"loss/base_kto": 3.8170158863067627,
|
|
"metrics/advantage_var": 266.03265380859375,
|
|
"regularization/mmd": 0.9423967599868774,
|
|
"regularization/rkhs_norm": 181.579345703125,
|
|
"rewards/chosen": 0.05191928717740782,
|
|
"rewards/margins": 0.16989548168783847,
|
|
"rewards/rejected": -0.11797619451043065,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 16.6367244720459,
|
|
"kl": 6.752360820770264,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35805528.523364484,
|
|
"logits/rejected": -36757918.09404389,
|
|
"logps/chosen": -476.0342192367601,
|
|
"logps/rejected": -365.6922756661442,
|
|
"loss": 0.5741,
|
|
"loss/base_kto": 3.688326358795166,
|
|
"metrics/advantage_var": 238.8223876953125,
|
|
"regularization/mmd": 0.9042009711265564,
|
|
"regularization/rkhs_norm": 174.21983337402344,
|
|
"rewards/chosen": 0.15758937467295805,
|
|
"rewards/margins": 0.3021852951519941,
|
|
"rewards/rejected": -0.14459592047903605,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 17.560890197753906,
|
|
"kl": 9.86482048034668,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -35733777.710691825,
|
|
"logits/rejected": -36475484.22360248,
|
|
"logps/chosen": -473.0930129716981,
|
|
"logps/rejected": -381.6515916149068,
|
|
"loss": 0.581,
|
|
"loss/base_kto": 3.7668347358703613,
|
|
"metrics/advantage_var": 221.4756317138672,
|
|
"regularization/mmd": 0.8811573386192322,
|
|
"regularization/rkhs_norm": 169.7798309326172,
|
|
"rewards/chosen": 0.07517084085716391,
|
|
"rewards/margins": 0.19803682575608061,
|
|
"rewards/rejected": -0.12286598489891669,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 12.482565879821777,
|
|
"kl": 16.427331924438477,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35223902.72782875,
|
|
"logits/rejected": -35434437.111821085,
|
|
"logps/chosen": -471.33720374617735,
|
|
"logps/rejected": -371.2310553115016,
|
|
"loss": 0.5728,
|
|
"loss/base_kto": 3.7031056880950928,
|
|
"metrics/advantage_var": 218.29490661621094,
|
|
"regularization/mmd": 0.8796938061714172,
|
|
"regularization/rkhs_norm": 169.4978485107422,
|
|
"rewards/chosen": 0.2948127303283878,
|
|
"rewards/margins": 0.28652009339611223,
|
|
"rewards/rejected": 0.008292636932275547,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 19.035781860351562,
|
|
"kl": 12.132369041442871,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36218929.46859904,
|
|
"logits/rejected": -36743385.5417299,
|
|
"logps/chosen": -460.4609500805153,
|
|
"logps/rejected": -368.035375569044,
|
|
"loss": 0.5769,
|
|
"loss/base_kto": 3.7111313343048096,
|
|
"metrics/advantage_var": 236.9758758544922,
|
|
"regularization/mmd": 0.9044443964958191,
|
|
"regularization/rkhs_norm": 174.26675415039062,
|
|
"rewards/chosen": 0.15490230001305227,
|
|
"rewards/margins": 0.26908599508696873,
|
|
"rewards/rejected": -0.11418369507391644,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 16.07984161376953,
|
|
"kl": 12.516695022583008,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36478836.01783061,
|
|
"logits/rejected": -38328242.39868204,
|
|
"logps/chosen": -502.0854383358098,
|
|
"logps/rejected": -379.00779962932455,
|
|
"loss": 0.5774,
|
|
"loss/base_kto": 3.7033092975616455,
|
|
"metrics/advantage_var": 255.03062438964844,
|
|
"regularization/mmd": 0.9161953926086426,
|
|
"regularization/rkhs_norm": 176.53091430664062,
|
|
"rewards/chosen": 0.2599937586337992,
|
|
"rewards/margins": 0.2863752622464369,
|
|
"rewards/rejected": -0.026381503612637715,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.324129104614258,
|
|
"kl": 9.51673412322998,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36140524.8,
|
|
"logits/rejected": -37949420.8,
|
|
"logps/chosen": -487.704736328125,
|
|
"logps/rejected": -384.366455078125,
|
|
"loss": 0.5837,
|
|
"loss/base_kto": 3.736816883087158,
|
|
"metrics/advantage_var": 251.3058624267578,
|
|
"regularization/mmd": 0.9326297640800476,
|
|
"regularization/rkhs_norm": 179.6974639892578,
|
|
"rewards/chosen": 0.11220818758010864,
|
|
"rewards/margins": 0.22923604249954224,
|
|
"rewards/rejected": -0.1170278549194336,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 15.10949420928955,
|
|
"kl": 7.855568885803223,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35668759.127725855,
|
|
"logits/rejected": -37519687.42319749,
|
|
"logps/chosen": -473.3497371495327,
|
|
"logps/rejected": -355.66881367554856,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.7436959743499756,
|
|
"metrics/advantage_var": 280.8514709472656,
|
|
"regularization/mmd": 0.9414481520652771,
|
|
"regularization/rkhs_norm": 181.3965606689453,
|
|
"rewards/chosen": 0.12582903710481164,
|
|
"rewards/margins": 0.2381549625431933,
|
|
"rewards/rejected": -0.11232592543838166,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 15.881586074829102,
|
|
"kl": 13.679503440856934,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36254788.69266771,
|
|
"logits/rejected": -38482833.427230045,
|
|
"logps/chosen": -490.9366224648986,
|
|
"logps/rejected": -356.3986697965571,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.7355542182922363,
|
|
"metrics/advantage_var": 268.415771484375,
|
|
"regularization/mmd": 0.9695192575454712,
|
|
"regularization/rkhs_norm": 186.8052520751953,
|
|
"rewards/chosen": 0.244347916006484,
|
|
"rewards/margins": 0.23980130205953054,
|
|
"rewards/rejected": 0.004546613946953476,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 14.873223304748535,
|
|
"kl": 11.618257522583008,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37046560.04945904,
|
|
"logits/rejected": -37583783.02685624,
|
|
"logps/chosen": -493.97198608964453,
|
|
"logps/rejected": -364.0125394944708,
|
|
"loss": 0.5783,
|
|
"loss/base_kto": 3.7623164653778076,
|
|
"metrics/advantage_var": 207.55296325683594,
|
|
"regularization/mmd": 0.864192008972168,
|
|
"regularization/rkhs_norm": 166.51101684570312,
|
|
"rewards/chosen": 0.10723678848290186,
|
|
"rewards/margins": 0.20618302687561574,
|
|
"rewards/rejected": -0.09894623839271388,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 15.5108003616333,
|
|
"kl": 15.256669998168945,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35425102.58966565,
|
|
"logits/rejected": -37325359.74276527,
|
|
"logps/chosen": -465.72791603343467,
|
|
"logps/rejected": -376.3284515675241,
|
|
"loss": 0.5781,
|
|
"loss/base_kto": 3.698437452316284,
|
|
"metrics/advantage_var": 239.4070587158203,
|
|
"regularization/mmd": 0.9260997772216797,
|
|
"regularization/rkhs_norm": 178.4392547607422,
|
|
"rewards/chosen": 0.30974333047142144,
|
|
"rewards/margins": 0.25810140013353394,
|
|
"rewards/rejected": 0.051641930337887484,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 21.99886131286621,
|
|
"kl": 7.631244659423828,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -35972634.0338983,
|
|
"logits/rejected": -37005282.789223455,
|
|
"logps/chosen": -483.33484206471496,
|
|
"logps/rejected": -385.09984152139464,
|
|
"loss": 0.5807,
|
|
"loss/base_kto": 3.710510730743408,
|
|
"metrics/advantage_var": 263.571044921875,
|
|
"regularization/mmd": 0.9350858926773071,
|
|
"regularization/rkhs_norm": 180.17068481445312,
|
|
"rewards/chosen": 0.13394736361981174,
|
|
"rewards/margins": 0.2811491677068264,
|
|
"rewards/rejected": -0.14720180408701466,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 16.300622940063477,
|
|
"kl": 17.796476364135742,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -34486187.74683544,
|
|
"logits/rejected": -35447084.24691358,
|
|
"logps/chosen": -471.895272943038,
|
|
"logps/rejected": -369.14689429012344,
|
|
"loss": 0.5823,
|
|
"loss/base_kto": 3.7178444862365723,
|
|
"metrics/advantage_var": 250.9248046875,
|
|
"regularization/mmd": 0.9402643442153931,
|
|
"regularization/rkhs_norm": 181.16845703125,
|
|
"rewards/chosen": 0.26546992531305624,
|
|
"rewards/margins": 0.22068698653244823,
|
|
"rewards/rejected": 0.044782938780608,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 18.066877365112305,
|
|
"kl": 13.168902397155762,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35066087.122257054,
|
|
"logits/rejected": -36025867.16510903,
|
|
"logps/chosen": -466.1370493730407,
|
|
"logps/rejected": -354.39505451713393,
|
|
"loss": 0.5882,
|
|
"loss/base_kto": 3.8002800941467285,
|
|
"metrics/advantage_var": 233.62783813476562,
|
|
"regularization/mmd": 0.9055353403091431,
|
|
"regularization/rkhs_norm": 174.47695922851562,
|
|
"rewards/chosen": 0.13914042505724677,
|
|
"rewards/margins": 0.17859217866763788,
|
|
"rewards/rejected": -0.03945175361039111,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 15.254158973693848,
|
|
"kl": 5.895008563995361,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36683634.09133858,
|
|
"logits/rejected": -36912262.94573643,
|
|
"logps/chosen": -494.22691929133856,
|
|
"logps/rejected": -368.55382751937987,
|
|
"loss": 0.5798,
|
|
"loss/base_kto": 3.7205963134765625,
|
|
"metrics/advantage_var": 235.41050720214844,
|
|
"regularization/mmd": 0.9181786775588989,
|
|
"regularization/rkhs_norm": 176.9130401611328,
|
|
"rewards/chosen": 0.07600033977838952,
|
|
"rewards/margins": 0.26991844668548737,
|
|
"rewards/rejected": -0.19391810690709788,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 19.226804733276367,
|
|
"kl": 14.0280179977417,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35125136.522550546,
|
|
"logits/rejected": -37260231.73626374,
|
|
"logps/chosen": -476.140066096423,
|
|
"logps/rejected": -388.09259713500785,
|
|
"loss": 0.576,
|
|
"loss/base_kto": 3.7070813179016113,
|
|
"metrics/advantage_var": 227.0825653076172,
|
|
"regularization/mmd": 0.9011588096618652,
|
|
"regularization/rkhs_norm": 173.63369750976562,
|
|
"rewards/chosen": 0.18713741984745577,
|
|
"rewards/margins": 0.2513987140821343,
|
|
"rewards/rejected": -0.06426129423467855,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 13.873737335205078,
|
|
"kl": 12.9469575881958,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -34681182.70759289,
|
|
"logits/rejected": -35259774.64447806,
|
|
"logps/chosen": -476.08678311793216,
|
|
"logps/rejected": -360.8672465960666,
|
|
"loss": 0.5726,
|
|
"loss/base_kto": 3.683901309967041,
|
|
"metrics/advantage_var": 236.10203552246094,
|
|
"regularization/mmd": 0.8970678448677063,
|
|
"regularization/rkhs_norm": 172.84544372558594,
|
|
"rewards/chosen": 0.25019873401845216,
|
|
"rewards/margins": 0.28955164155416097,
|
|
"rewards/rejected": -0.039352907535708796,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 14.483478546142578,
|
|
"kl": 7.9873785972595215,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36476209.45904173,
|
|
"logits/rejected": -37305842.20602219,
|
|
"logps/chosen": -505.45319744976814,
|
|
"logps/rejected": -376.2053783676704,
|
|
"loss": 0.5841,
|
|
"loss/base_kto": 3.5255863666534424,
|
|
"metrics/advantage_var": 409.21575927734375,
|
|
"regularization/mmd": 1.1474695205688477,
|
|
"regularization/rkhs_norm": 221.09239196777344,
|
|
"rewards/chosen": 0.23478642129087374,
|
|
"rewards/margins": 0.5181088356830217,
|
|
"rewards/rejected": -0.2833224143921479,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 11.03209114074707,
|
|
"kl": 9.305520057678223,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35367011.2,
|
|
"logits/rejected": -36813363.2,
|
|
"logps/chosen": -476.743505859375,
|
|
"logps/rejected": -373.8144775390625,
|
|
"loss": 0.5677,
|
|
"loss/base_kto": 3.2241909503936768,
|
|
"metrics/advantage_var": 555.5560913085938,
|
|
"regularization/mmd": 1.3173788785934448,
|
|
"regularization/rkhs_norm": 253.83023071289062,
|
|
"rewards/chosen": 0.4903388977050781,
|
|
"rewards/margins": 0.8678805112838746,
|
|
"rewards/rejected": -0.3775416135787964,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 14.804159164428711,
|
|
"kl": 13.329559326171875,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35032850.638930164,
|
|
"logits/rejected": -36724776.48764415,
|
|
"logps/chosen": -484.1767737741456,
|
|
"logps/rejected": -374.4861511532125,
|
|
"loss": 0.5617,
|
|
"loss/base_kto": 3.169060230255127,
|
|
"metrics/advantage_var": 662.4751586914062,
|
|
"regularization/mmd": 1.3242170810699463,
|
|
"regularization/rkhs_norm": 255.14781188964844,
|
|
"rewards/chosen": 0.5468870619253807,
|
|
"rewards/margins": 0.9381494368787245,
|
|
"rewards/rejected": -0.3912623749533438,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 10.822148323059082,
|
|
"kl": 17.54160499572754,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36543880.101265825,
|
|
"logits/rejected": -36830517.72839506,
|
|
"logps/chosen": -472.810176028481,
|
|
"logps/rejected": -364.96195023148147,
|
|
"loss": 0.5547,
|
|
"loss/base_kto": 3.14931321144104,
|
|
"metrics/advantage_var": 500.598876953125,
|
|
"regularization/mmd": 1.2882076501846313,
|
|
"regularization/rkhs_norm": 248.20957946777344,
|
|
"rewards/chosen": 0.6308662800849238,
|
|
"rewards/margins": 0.9442392806184163,
|
|
"rewards/rejected": -0.3133730005334925,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 11.496271133422852,
|
|
"kl": 16.021413803100586,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34998066.5775076,
|
|
"logits/rejected": -37187412.78456592,
|
|
"logps/chosen": -488.53804141337383,
|
|
"logps/rejected": -373.63987138263667,
|
|
"loss": 0.5752,
|
|
"loss/base_kto": 3.2147233486175537,
|
|
"metrics/advantage_var": 640.8287963867188,
|
|
"regularization/mmd": 1.3867143392562866,
|
|
"regularization/rkhs_norm": 267.189697265625,
|
|
"rewards/chosen": 0.578866326700228,
|
|
"rewards/margins": 0.8966528347513735,
|
|
"rewards/rejected": -0.3177865080511455,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 17.11268424987793,
|
|
"kl": 13.3150053024292,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -33797192.46769231,
|
|
"logits/rejected": -36451581.56190476,
|
|
"logps/chosen": -490.08384615384614,
|
|
"logps/rejected": -360.4661706349206,
|
|
"loss": 0.562,
|
|
"loss/base_kto": 3.1174914836883545,
|
|
"metrics/advantage_var": 571.0593872070312,
|
|
"regularization/mmd": 1.378857970237732,
|
|
"regularization/rkhs_norm": 265.6759033203125,
|
|
"rewards/chosen": 0.6120276817908654,
|
|
"rewards/margins": 0.9400860033046923,
|
|
"rewards/rejected": -0.3280583215138269,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 11.711071968078613,
|
|
"kl": 14.93961238861084,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34375409.033846155,
|
|
"logits/rejected": -36298563.45396826,
|
|
"logps/chosen": -487.9735576923077,
|
|
"logps/rejected": -358.37366071428573,
|
|
"loss": 0.565,
|
|
"loss/base_kto": 3.148419141769409,
|
|
"metrics/advantage_var": 573.4874877929688,
|
|
"regularization/mmd": 1.37118399143219,
|
|
"regularization/rkhs_norm": 264.19732666015625,
|
|
"rewards/chosen": 0.5945619553786058,
|
|
"rewards/margins": 0.930740495805193,
|
|
"rewards/rejected": -0.3361785404265873,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 14.130123138427734,
|
|
"kl": 9.879761695861816,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34500308.29268292,
|
|
"logits/rejected": -37149728.336842105,
|
|
"logps/chosen": -474.78231707317076,
|
|
"logps/rejected": -395.69891917293234,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.209912061691284,
|
|
"metrics/advantage_var": 538.355712890625,
|
|
"regularization/mmd": 1.2786645889282227,
|
|
"regularization/rkhs_norm": 246.37083435058594,
|
|
"rewards/chosen": 0.3794711725498603,
|
|
"rewards/margins": 0.8551711035296535,
|
|
"rewards/rejected": -0.47569993097979324,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 13.540090560913086,
|
|
"kl": 5.989752292633057,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34450699.50083195,
|
|
"logits/rejected": -35318646.7628866,
|
|
"logps/chosen": -492.9532029950083,
|
|
"logps/rejected": -387.5404086892489,
|
|
"loss": 0.5696,
|
|
"loss/base_kto": 3.2576324939727783,
|
|
"metrics/advantage_var": 524.0263061523438,
|
|
"regularization/mmd": 1.2991846799850464,
|
|
"regularization/rkhs_norm": 250.32461547851562,
|
|
"rewards/chosen": 0.3055762165596402,
|
|
"rewards/margins": 0.795464426041645,
|
|
"rewards/rejected": -0.4898882094820048,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 10.40711784362793,
|
|
"kl": 9.336389541625977,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35843078.440251574,
|
|
"logits/rejected": -37510671.900621116,
|
|
"logps/chosen": -495.6369889937107,
|
|
"logps/rejected": -369.85721564440996,
|
|
"loss": 0.5743,
|
|
"loss/base_kto": 3.193208932876587,
|
|
"metrics/advantage_var": 623.3612060546875,
|
|
"regularization/mmd": 1.401095986366272,
|
|
"regularization/rkhs_norm": 269.9606628417969,
|
|
"rewards/chosen": 0.383165827337301,
|
|
"rewards/margins": 0.8949085554343992,
|
|
"rewards/rejected": -0.5117427280970982,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 14.492140769958496,
|
|
"kl": 11.189831733703613,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35120141.2987013,
|
|
"logits/rejected": -37403182.26506024,
|
|
"logps/chosen": -501.71935876623377,
|
|
"logps/rejected": -403.6578501506024,
|
|
"loss": 0.5654,
|
|
"loss/base_kto": 3.1605660915374756,
|
|
"metrics/advantage_var": 598.8322143554688,
|
|
"regularization/mmd": 1.3623825311660767,
|
|
"regularization/rkhs_norm": 262.5014343261719,
|
|
"rewards/chosen": 0.4411466524198458,
|
|
"rewards/margins": 0.9260114503567105,
|
|
"rewards/rejected": -0.48486479793686466,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 14.895859718322754,
|
|
"kl": 5.659992218017578,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34601870.9202454,
|
|
"logits/rejected": -35387046.318471335,
|
|
"logps/chosen": -481.83291794478527,
|
|
"logps/rejected": -385.2407941878981,
|
|
"loss": 0.571,
|
|
"loss/base_kto": 3.239588499069214,
|
|
"metrics/advantage_var": 549.1942749023438,
|
|
"regularization/mmd": 1.3287962675094604,
|
|
"regularization/rkhs_norm": 256.0301208496094,
|
|
"rewards/chosen": 0.3637069514924032,
|
|
"rewards/margins": 0.8523527602043359,
|
|
"rewards/rejected": -0.48864580871193275,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 14.638245582580566,
|
|
"kl": 17.762792587280273,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35373422.15757576,
|
|
"logits/rejected": -37802558.76129032,
|
|
"logps/chosen": -491.02272727272725,
|
|
"logps/rejected": -361.95703125,
|
|
"loss": 0.5642,
|
|
"loss/base_kto": 3.2021262645721436,
|
|
"metrics/advantage_var": 522.0460205078125,
|
|
"regularization/mmd": 1.3116295337677002,
|
|
"regularization/rkhs_norm": 252.7224578857422,
|
|
"rewards/chosen": 0.6330557158499053,
|
|
"rewards/margins": 0.8610706064591542,
|
|
"rewards/rejected": -0.22801489060924898,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 15.502215385437012,
|
|
"kl": 12.964693069458008,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35737295.18139535,
|
|
"logits/rejected": -37757989.08976378,
|
|
"logps/chosen": -491.22093023255815,
|
|
"logps/rejected": -369.54714566929135,
|
|
"loss": 0.5683,
|
|
"loss/base_kto": 3.2542412281036377,
|
|
"metrics/advantage_var": 542.7702026367188,
|
|
"regularization/mmd": 1.291926622390747,
|
|
"regularization/rkhs_norm": 248.92613220214844,
|
|
"rewards/chosen": 0.512014652222626,
|
|
"rewards/margins": 0.8132746677553524,
|
|
"rewards/rejected": -0.30126001553272636,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.078570365905762,
|
|
"kl": 8.791579246520996,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35871753.69085173,
|
|
"logits/rejected": -37875309.374613,
|
|
"logps/chosen": -490.7807570977918,
|
|
"logps/rejected": -354.6810903637771,
|
|
"loss": 0.5692,
|
|
"loss/base_kto": 3.150142192840576,
|
|
"metrics/advantage_var": 969.3246459960938,
|
|
"regularization/mmd": 1.403619647026062,
|
|
"regularization/rkhs_norm": 270.44696044921875,
|
|
"rewards/chosen": 0.48477798606319006,
|
|
"rewards/margins": 0.8774978808030556,
|
|
"rewards/rejected": -0.3927198947398655,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 10.45084285736084,
|
|
"kl": 14.742172241210938,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34520927.24646782,
|
|
"logits/rejected": -35291588.27993779,
|
|
"logps/chosen": -470.88000392464676,
|
|
"logps/rejected": -392.1973658631415,
|
|
"loss": 0.5621,
|
|
"loss/base_kto": 3.209953784942627,
|
|
"metrics/advantage_var": 507.49639892578125,
|
|
"regularization/mmd": 1.2871525287628174,
|
|
"regularization/rkhs_norm": 248.00625610351562,
|
|
"rewards/chosen": 0.4810376129869015,
|
|
"rewards/margins": 0.8173346081587644,
|
|
"rewards/rejected": -0.33629699517186284,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 11.959739685058594,
|
|
"kl": 10.621394157409668,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36385666.51739788,
|
|
"logits/rejected": -36326851.6187399,
|
|
"logps/chosen": -468.92274962178516,
|
|
"logps/rejected": -365.3991569063005,
|
|
"loss": 0.5649,
|
|
"loss/base_kto": 3.174787998199463,
|
|
"metrics/advantage_var": 563.9235229492188,
|
|
"regularization/mmd": 1.3442481756210327,
|
|
"regularization/rkhs_norm": 259.00738525390625,
|
|
"rewards/chosen": 0.5609467429941614,
|
|
"rewards/margins": 0.9389897991062752,
|
|
"rewards/rejected": -0.3780430561121138,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 15.207474708557129,
|
|
"kl": 11.307988166809082,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -36114899.37242472,
|
|
"logits/rejected": -37777389.06625578,
|
|
"logps/chosen": -478.51361925515056,
|
|
"logps/rejected": -401.49157357473035,
|
|
"loss": 0.5768,
|
|
"loss/base_kto": 3.2136363983154297,
|
|
"metrics/advantage_var": 624.7119140625,
|
|
"regularization/mmd": 1.4008406400680542,
|
|
"regularization/rkhs_norm": 269.9114685058594,
|
|
"rewards/chosen": 0.41678248192353406,
|
|
"rewards/margins": 0.8572625775032721,
|
|
"rewards/rejected": -0.44048009557973805,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 9.94320297241211,
|
|
"kl": 22.93610954284668,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -36681682.776025236,
|
|
"logits/rejected": -36529548.28482972,
|
|
"logps/chosen": -471.74773264984225,
|
|
"logps/rejected": -379.4496662151703,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.148862838745117,
|
|
"metrics/advantage_var": 561.9501953125,
|
|
"regularization/mmd": 1.3347347974777222,
|
|
"regularization/rkhs_norm": 257.17431640625,
|
|
"rewards/chosen": 0.6471507556806979,
|
|
"rewards/margins": 0.9064340990164903,
|
|
"rewards/rejected": -0.2592833433357924,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 11.12475872039795,
|
|
"kl": 15.809300422668457,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34522435.368421055,
|
|
"logits/rejected": -36607589.75394322,
|
|
"logps/chosen": -505.07410990712077,
|
|
"logps/rejected": -361.7321322949527,
|
|
"loss": 0.5637,
|
|
"loss/base_kto": 3.187002658843994,
|
|
"metrics/advantage_var": 580.5582885742188,
|
|
"regularization/mmd": 1.3225512504577637,
|
|
"regularization/rkhs_norm": 254.82681274414062,
|
|
"rewards/chosen": 0.5610994037828947,
|
|
"rewards/margins": 0.9047024958200596,
|
|
"rewards/rejected": -0.3436030920371648,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 15.943488121032715,
|
|
"kl": 12.461132049560547,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -36111878.64935065,
|
|
"logits/rejected": -36577773.4939759,
|
|
"logps/chosen": -490.8744419642857,
|
|
"logps/rejected": -393.5507577183735,
|
|
"loss": 0.5517,
|
|
"loss/base_kto": 3.2027695178985596,
|
|
"metrics/advantage_var": 430.74444580078125,
|
|
"regularization/mmd": 1.2105010747909546,
|
|
"regularization/rkhs_norm": 233.23721313476562,
|
|
"rewards/chosen": 0.4537938601010806,
|
|
"rewards/margins": 0.8291066193166539,
|
|
"rewards/rejected": -0.3753127592155732,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 19.020092010498047,
|
|
"kl": 8.088566780090332,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33876174.80130293,
|
|
"logits/rejected": -35707811.74774775,
|
|
"logps/chosen": -498.2650651465798,
|
|
"logps/rejected": -364.7563344594595,
|
|
"loss": 0.5748,
|
|
"loss/base_kto": 3.2149109840393066,
|
|
"metrics/advantage_var": 621.2483520507812,
|
|
"regularization/mmd": 1.3834377527236938,
|
|
"regularization/rkhs_norm": 266.5583190917969,
|
|
"rewards/chosen": 0.383180301430171,
|
|
"rewards/margins": 0.8799245432305651,
|
|
"rewards/rejected": -0.49674424180039417,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 9.668807029724121,
|
|
"kl": 14.72839641571045,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -36003460.62166405,
|
|
"logits/rejected": -36985449.903576985,
|
|
"logps/chosen": -470.77315541601257,
|
|
"logps/rejected": -384.33944887247276,
|
|
"loss": 0.5584,
|
|
"loss/base_kto": 3.1495158672332764,
|
|
"metrics/advantage_var": 551.7531127929688,
|
|
"regularization/mmd": 1.3180047273635864,
|
|
"regularization/rkhs_norm": 253.95083618164062,
|
|
"rewards/chosen": 0.5112585909138295,
|
|
"rewards/margins": 0.9135487097112148,
|
|
"rewards/rejected": -0.4022901187973853,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 46.552490234375,
|
|
"kl": 10.026848793029785,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34977852.43278688,
|
|
"logits/rejected": -35357549.27761194,
|
|
"logps/chosen": -452.09149590163935,
|
|
"logps/rejected": -358.71585820895524,
|
|
"loss": 0.569,
|
|
"loss/base_kto": 3.180851697921753,
|
|
"metrics/advantage_var": 694.2982177734375,
|
|
"regularization/mmd": 1.3709758520126343,
|
|
"regularization/rkhs_norm": 264.1571960449219,
|
|
"rewards/chosen": 0.41222759309362195,
|
|
"rewards/margins": 0.8554363387938319,
|
|
"rewards/rejected": -0.4432087457002099,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 14.278236389160156,
|
|
"kl": 14.127997398376465,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34378638.901492536,
|
|
"logits/rejected": -35840933.35081967,
|
|
"logps/chosen": -477.92793843283584,
|
|
"logps/rejected": -385.291931352459,
|
|
"loss": 0.5693,
|
|
"loss/base_kto": 3.2170772552490234,
|
|
"metrics/advantage_var": 563.250244140625,
|
|
"regularization/mmd": 1.3377063274383545,
|
|
"regularization/rkhs_norm": 257.74688720703125,
|
|
"rewards/chosen": 0.519941552005597,
|
|
"rewards/margins": 0.8542181453233479,
|
|
"rewards/rejected": -0.334276593317751,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 11.11368179321289,
|
|
"kl": 18.76359748840332,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34820701.01737757,
|
|
"logits/rejected": -34516112.02472952,
|
|
"logps/chosen": -446.1201125592417,
|
|
"logps/rejected": -352.66953245749613,
|
|
"loss": 0.5618,
|
|
"loss/base_kto": 3.2049381732940674,
|
|
"metrics/advantage_var": 503.1366882324219,
|
|
"regularization/mmd": 1.2894788980484009,
|
|
"regularization/rkhs_norm": 248.4545135498047,
|
|
"rewards/chosen": 0.6195756812796208,
|
|
"rewards/margins": 0.8582325286128656,
|
|
"rewards/rejected": -0.23865684733324477,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 11.907867431640625,
|
|
"kl": 8.349406242370605,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33516892.870662462,
|
|
"logits/rejected": -34123313.13931888,
|
|
"logps/chosen": -463.0879337539432,
|
|
"logps/rejected": -351.16802921826627,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.2693023681640625,
|
|
"metrics/advantage_var": 432.8820495605469,
|
|
"regularization/mmd": 1.1837867498397827,
|
|
"regularization/rkhs_norm": 228.08995056152344,
|
|
"rewards/chosen": 0.3898502434089733,
|
|
"rewards/margins": 0.7927915427057273,
|
|
"rewards/rejected": -0.40294129929675404,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 19.638782501220703,
|
|
"kl": 10.250836372375488,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34289782.53376206,
|
|
"logits/rejected": -35860732.10942249,
|
|
"logps/chosen": -482.92614549839226,
|
|
"logps/rejected": -387.59719319908817,
|
|
"loss": 0.5681,
|
|
"loss/base_kto": 3.2245795726776123,
|
|
"metrics/advantage_var": 564.8365478515625,
|
|
"regularization/mmd": 1.3204599618911743,
|
|
"regularization/rkhs_norm": 254.4238739013672,
|
|
"rewards/chosen": 0.40070024103980356,
|
|
"rewards/margins": 0.8202269644203126,
|
|
"rewards/rejected": -0.4195267233805091,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 13.80681324005127,
|
|
"kl": 9.39556884765625,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35517312.192481205,
|
|
"logits/rejected": -36264885.07317073,
|
|
"logps/chosen": -496.2058270676692,
|
|
"logps/rejected": -379.8111788617886,
|
|
"loss": 0.5666,
|
|
"loss/base_kto": 3.2209389209747314,
|
|
"metrics/advantage_var": 521.9708251953125,
|
|
"regularization/mmd": 1.312155842781067,
|
|
"regularization/rkhs_norm": 252.82388305664062,
|
|
"rewards/chosen": 0.4581374548431626,
|
|
"rewards/margins": 0.8154349769150631,
|
|
"rewards/rejected": -0.3572975220719004,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 13.431105613708496,
|
|
"kl": 11.714164733886719,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35926315.251141556,
|
|
"logits/rejected": -36230875.428571425,
|
|
"logps/chosen": -507.626997716895,
|
|
"logps/rejected": -377.32115268860355,
|
|
"loss": 0.5645,
|
|
"loss/base_kto": 3.2468605041503906,
|
|
"metrics/advantage_var": 504.2196960449219,
|
|
"regularization/mmd": 1.2694696187973022,
|
|
"regularization/rkhs_norm": 244.59915161132812,
|
|
"rewards/chosen": 0.5132574955259466,
|
|
"rewards/margins": 0.7778916784268545,
|
|
"rewards/rejected": -0.2646341829009079,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 11.592087745666504,
|
|
"kl": 12.167591094970703,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34635315.2,
|
|
"logits/rejected": -35708259.2,
|
|
"logps/chosen": -470.50869140625,
|
|
"logps/rejected": -374.138134765625,
|
|
"loss": 0.5728,
|
|
"loss/base_kto": 3.1993470191955566,
|
|
"metrics/advantage_var": 635.9799194335938,
|
|
"regularization/mmd": 1.3829056024551392,
|
|
"regularization/rkhs_norm": 266.455810546875,
|
|
"rewards/chosen": 0.4886659622192383,
|
|
"rewards/margins": 0.8927920818328858,
|
|
"rewards/rejected": -0.40412611961364747,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.77651023864746,
|
|
"kl": 10.939823150634766,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34838283.80286168,
|
|
"logits/rejected": -37276078.99231951,
|
|
"logps/chosen": -466.2969495230525,
|
|
"logps/rejected": -381.3138440860215,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.169776678085327,
|
|
"metrics/advantage_var": 550.9974975585938,
|
|
"regularization/mmd": 1.2822946310043335,
|
|
"regularization/rkhs_norm": 247.07029724121094,
|
|
"rewards/chosen": 0.4813091720799384,
|
|
"rewards/margins": 0.9120447704082314,
|
|
"rewards/rejected": -0.430735598328293,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 14.149236679077148,
|
|
"kl": 10.224510192871094,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36272958.52534562,
|
|
"logits/rejected": -36580581.54531001,
|
|
"logps/chosen": -474.05721966205834,
|
|
"logps/rejected": -389.4228686406995,
|
|
"loss": 0.5606,
|
|
"loss/base_kto": 3.1691484451293945,
|
|
"metrics/advantage_var": 546.0302734375,
|
|
"regularization/mmd": 1.3153948783874512,
|
|
"regularization/rkhs_norm": 253.4479522705078,
|
|
"rewards/chosen": 0.49803646808395735,
|
|
"rewards/margins": 0.9197941524974609,
|
|
"rewards/rejected": -0.4217576844135036,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 12.257226943969727,
|
|
"kl": 16.28075408935547,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -36167804.5184,
|
|
"logits/rejected": -36414096.610687025,
|
|
"logps/chosen": -457.8833,
|
|
"logps/rejected": -380.27562022900764,
|
|
"loss": 0.5725,
|
|
"loss/base_kto": 3.246429443359375,
|
|
"metrics/advantage_var": 582.8483276367188,
|
|
"regularization/mmd": 1.3332990407943726,
|
|
"regularization/rkhs_norm": 256.897705078125,
|
|
"rewards/chosen": 0.505484912109375,
|
|
"rewards/margins": 0.8409347365517652,
|
|
"rewards/rejected": -0.33544982444239024,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 16.778846740722656,
|
|
"kl": 12.644331932067871,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34810601.3164557,
|
|
"logits/rejected": -36375523.55555555,
|
|
"logps/chosen": -483.8537381329114,
|
|
"logps/rejected": -379.38635706018516,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.255591630935669,
|
|
"metrics/advantage_var": 539.2044677734375,
|
|
"regularization/mmd": 1.2822104692459106,
|
|
"regularization/rkhs_norm": 247.0540313720703,
|
|
"rewards/chosen": 0.45679077921034417,
|
|
"rewards/margins": 0.7998983559337962,
|
|
"rewards/rejected": -0.343107576723452,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 10.346834182739258,
|
|
"kl": 12.630763053894043,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36129978.90795632,
|
|
"logits/rejected": -36763853.12050078,
|
|
"logps/chosen": -500.6096918876755,
|
|
"logps/rejected": -381.59634194053206,
|
|
"loss": 0.5595,
|
|
"loss/base_kto": 3.1904945373535156,
|
|
"metrics/advantage_var": 562.1464233398438,
|
|
"regularization/mmd": 1.2853113412857056,
|
|
"regularization/rkhs_norm": 247.65151977539062,
|
|
"rewards/chosen": 0.5136202188810208,
|
|
"rewards/margins": 0.8641619082806564,
|
|
"rewards/rejected": -0.35054168939963565,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 15.149313926696777,
|
|
"kl": 14.863423347473145,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35679784.358578056,
|
|
"logits/rejected": -37479060.01895735,
|
|
"logps/chosen": -484.0900792117465,
|
|
"logps/rejected": -370.00083925750397,
|
|
"loss": 0.5752,
|
|
"loss/base_kto": 3.2095329761505127,
|
|
"metrics/advantage_var": 710.6679077148438,
|
|
"regularization/mmd": 1.3923863172531128,
|
|
"regularization/rkhs_norm": 268.28253173828125,
|
|
"rewards/chosen": 0.5660976309680497,
|
|
"rewards/margins": 0.8733764735023605,
|
|
"rewards/rejected": -0.3072788425343108,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 12.813681602478027,
|
|
"kl": 12.500152587890625,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33813792.49535604,
|
|
"logits/rejected": -35740904.58044164,
|
|
"logps/chosen": -477.83813854489165,
|
|
"logps/rejected": -390.43276813880124,
|
|
"loss": 0.5578,
|
|
"loss/base_kto": 3.1661529541015625,
|
|
"metrics/advantage_var": 496.9759826660156,
|
|
"regularization/mmd": 1.2963389158248901,
|
|
"regularization/rkhs_norm": 249.77627563476562,
|
|
"rewards/chosen": 0.5235694436465993,
|
|
"rewards/margins": 0.8826646831354596,
|
|
"rewards/rejected": -0.3590952394888604,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 13.28852653503418,
|
|
"kl": 20.830835342407227,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34747848.48192771,
|
|
"logits/rejected": -35427341.2987013,
|
|
"logps/chosen": -459.82516001506025,
|
|
"logps/rejected": -382.30509334415586,
|
|
"loss": 0.5688,
|
|
"loss/base_kto": 3.1892714500427246,
|
|
"metrics/advantage_var": 584.1787109375,
|
|
"regularization/mmd": 1.3608081340789795,
|
|
"regularization/rkhs_norm": 262.1980895996094,
|
|
"rewards/chosen": 0.6476653685052711,
|
|
"rewards/margins": 0.8488772730557987,
|
|
"rewards/rejected": -0.2012119045505276,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 11.812023162841797,
|
|
"kl": 17.362991333007812,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34660524.53042122,
|
|
"logits/rejected": -36000039.78649922,
|
|
"logps/chosen": -485.6875975039002,
|
|
"logps/rejected": -363.7608418367347,
|
|
"loss": 0.5396,
|
|
"loss/base_kto": 3.0937564373016357,
|
|
"metrics/advantage_var": 465.77960205078125,
|
|
"regularization/mmd": 1.2231942415237427,
|
|
"regularization/rkhs_norm": 235.6829071044922,
|
|
"rewards/chosen": 0.686273155271914,
|
|
"rewards/margins": 0.964398600818973,
|
|
"rewards/rejected": -0.278125445547059,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 13.598008155822754,
|
|
"kl": 12.415908813476562,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35139206.4,
|
|
"logits/rejected": -35781446.4,
|
|
"logps/chosen": -468.61357421875,
|
|
"logps/rejected": -395.6599853515625,
|
|
"loss": 0.5442,
|
|
"loss/base_kto": 3.1144955158233643,
|
|
"metrics/advantage_var": 742.1534423828125,
|
|
"regularization/mmd": 1.2394458055496216,
|
|
"regularization/rkhs_norm": 238.814208984375,
|
|
"rewards/chosen": 0.5431042671203613,
|
|
"rewards/margins": 0.897245216369629,
|
|
"rewards/rejected": -0.3541409492492676,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 10.59531021118164,
|
|
"kl": 15.029504776000977,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34486255.64217252,
|
|
"logits/rejected": -36448938.666666664,
|
|
"logps/chosen": -480.65460263578274,
|
|
"logps/rejected": -389.69399847094803,
|
|
"loss": 0.5218,
|
|
"loss/base_kto": 3.02362322807312,
|
|
"metrics/advantage_var": 382.8936462402344,
|
|
"regularization/mmd": 1.1508859395980835,
|
|
"regularization/rkhs_norm": 221.7506561279297,
|
|
"rewards/chosen": 0.5973086616101737,
|
|
"rewards/margins": 0.9933514183236676,
|
|
"rewards/rejected": -0.39604275671349387,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 13.953946113586426,
|
|
"kl": 10.442084312438965,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35427420.50079239,
|
|
"logits/rejected": -36524697.8366718,
|
|
"logps/chosen": -498.19284865293184,
|
|
"logps/rejected": -387.099311440678,
|
|
"loss": 0.5242,
|
|
"loss/base_kto": 3.0552632808685303,
|
|
"metrics/advantage_var": 368.9991149902344,
|
|
"regularization/mmd": 1.1386114358901978,
|
|
"regularization/rkhs_norm": 219.3856201171875,
|
|
"rewards/chosen": 0.5488621731378764,
|
|
"rewards/margins": 0.965508839583851,
|
|
"rewards/rejected": -0.4166466664459746,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 13.173537254333496,
|
|
"kl": 13.659466743469238,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35595632.145015106,
|
|
"logits/rejected": -35881510.110032365,
|
|
"logps/chosen": -505.66550226586105,
|
|
"logps/rejected": -363.38647855987057,
|
|
"loss": 0.5277,
|
|
"loss/base_kto": 3.065598249435425,
|
|
"metrics/advantage_var": 378.0130310058594,
|
|
"regularization/mmd": 1.1558254957199097,
|
|
"regularization/rkhs_norm": 222.702392578125,
|
|
"rewards/chosen": 0.5783516229819675,
|
|
"rewards/margins": 0.9472788471504672,
|
|
"rewards/rejected": -0.3689272241684997,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 11.931377410888672,
|
|
"kl": 10.723335266113281,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34421327.92581144,
|
|
"logits/rejected": -36117059.1342812,
|
|
"logps/chosen": -458.4371136012365,
|
|
"logps/rejected": -395.20228574249603,
|
|
"loss": 0.5336,
|
|
"loss/base_kto": 3.097973346710205,
|
|
"metrics/advantage_var": 413.72314453125,
|
|
"regularization/mmd": 1.1707619428634644,
|
|
"regularization/rkhs_norm": 225.580322265625,
|
|
"rewards/chosen": 0.5291910304168277,
|
|
"rewards/margins": 0.9380135706518198,
|
|
"rewards/rejected": -0.4088225402349921,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 10.052943229675293,
|
|
"kl": 11.873779296875,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35897449.66312595,
|
|
"logits/rejected": -37358936.631239936,
|
|
"logps/chosen": -480.7319328528073,
|
|
"logps/rejected": -378.75747282608694,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.065223217010498,
|
|
"metrics/advantage_var": 365.3321228027344,
|
|
"regularization/mmd": 1.1347383260726929,
|
|
"regularization/rkhs_norm": 218.63938903808594,
|
|
"rewards/chosen": 0.5719722116843703,
|
|
"rewards/margins": 0.9500131581165361,
|
|
"rewards/rejected": -0.37804094643216585,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 11.726381301879883,
|
|
"kl": 12.947540283203125,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34227976.152866244,
|
|
"logits/rejected": -36331997.44785276,
|
|
"logps/chosen": -489.10091560509557,
|
|
"logps/rejected": -359.5610381518405,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.102567434310913,
|
|
"metrics/advantage_var": 368.41070556640625,
|
|
"regularization/mmd": 1.1274595260620117,
|
|
"regularization/rkhs_norm": 217.23692321777344,
|
|
"rewards/chosen": 0.5757263572352707,
|
|
"rewards/margins": 0.9222583714852945,
|
|
"rewards/rejected": -0.34653201425002395,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 11.503512382507324,
|
|
"kl": 13.424036026000977,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34937517.82716049,
|
|
"logits/rejected": -35918433.215189874,
|
|
"logps/chosen": -516.2775848765432,
|
|
"logps/rejected": -386.8339102056962,
|
|
"loss": 0.5292,
|
|
"loss/base_kto": 3.084811210632324,
|
|
"metrics/advantage_var": 368.7038879394531,
|
|
"regularization/mmd": 1.148498773574829,
|
|
"regularization/rkhs_norm": 221.2906951904297,
|
|
"rewards/chosen": 0.5573343347620081,
|
|
"rewards/margins": 0.9301758197010039,
|
|
"rewards/rejected": -0.37284148493899577,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 7.271444797515869,
|
|
"kl": 15.21033000946045,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34667300.00618238,
|
|
"logits/rejected": -35852171.52606635,
|
|
"logps/chosen": -471.90813369397216,
|
|
"logps/rejected": -373.40131319115324,
|
|
"loss": 0.5246,
|
|
"loss/base_kto": 3.0619912147521973,
|
|
"metrics/advantage_var": 368.8582458496094,
|
|
"regularization/mmd": 1.134839415550232,
|
|
"regularization/rkhs_norm": 218.65884399414062,
|
|
"rewards/chosen": 0.633236963192499,
|
|
"rewards/margins": 0.976519605526869,
|
|
"rewards/rejected": -0.34328264233437006,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 12.789861679077148,
|
|
"kl": 11.074790954589844,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35224399.11251981,
|
|
"logits/rejected": -37280491.882896766,
|
|
"logps/chosen": -475.46805665610145,
|
|
"logps/rejected": -370.1136363636364,
|
|
"loss": 0.5254,
|
|
"loss/base_kto": 3.0926616191864014,
|
|
"metrics/advantage_var": 383.5738525390625,
|
|
"regularization/mmd": 1.1108897924423218,
|
|
"regularization/rkhs_norm": 214.04429626464844,
|
|
"rewards/chosen": 0.544474896462708,
|
|
"rewards/margins": 0.9385429752076522,
|
|
"rewards/rejected": -0.39406807874494415,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 9.382735252380371,
|
|
"kl": 12.817514419555664,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34063755.90212072,
|
|
"logits/rejected": -36143705.811094455,
|
|
"logps/chosen": -472.3943719412724,
|
|
"logps/rejected": -364.0275487256372,
|
|
"loss": 0.5204,
|
|
"loss/base_kto": 3.0654656887054443,
|
|
"metrics/advantage_var": 342.53643798828125,
|
|
"regularization/mmd": 1.098124384880066,
|
|
"regularization/rkhs_norm": 211.5846710205078,
|
|
"rewards/chosen": 0.5396833061977467,
|
|
"rewards/margins": 0.9245164633496396,
|
|
"rewards/rejected": -0.3848331571518928,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 10.125913619995117,
|
|
"kl": 8.758418083190918,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34721252.71473354,
|
|
"logits/rejected": -35780895.10280374,
|
|
"logps/chosen": -482.1507641065831,
|
|
"logps/rejected": -382.7769421728972,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.088571548461914,
|
|
"metrics/advantage_var": 377.4120178222656,
|
|
"regularization/mmd": 1.1480095386505127,
|
|
"regularization/rkhs_norm": 221.19644165039062,
|
|
"rewards/chosen": 0.5008663069865547,
|
|
"rewards/margins": 0.9467187589646261,
|
|
"rewards/rejected": -0.44585245197807144,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 10.050665855407715,
|
|
"kl": 9.842815399169922,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36315312.71494607,
|
|
"logits/rejected": -36899818.90332805,
|
|
"logps/chosen": -493.86951078582433,
|
|
"logps/rejected": -396.1964391838352,
|
|
"loss": 0.5311,
|
|
"loss/base_kto": 3.056138753890991,
|
|
"metrics/advantage_var": 409.9040222167969,
|
|
"regularization/mmd": 1.1925348043441772,
|
|
"regularization/rkhs_norm": 229.7755126953125,
|
|
"rewards/chosen": 0.5343352848282213,
|
|
"rewards/margins": 0.9792982110490018,
|
|
"rewards/rejected": -0.4449629262207805,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 10.840620994567871,
|
|
"kl": 13.91283893585205,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33469242.273155417,
|
|
"logits/rejected": -35610678.146189734,
|
|
"logps/chosen": -486.79341640502355,
|
|
"logps/rejected": -389.954801710731,
|
|
"loss": 0.5374,
|
|
"loss/base_kto": 3.0902035236358643,
|
|
"metrics/advantage_var": 433.07330322265625,
|
|
"regularization/mmd": 1.209328293800354,
|
|
"regularization/rkhs_norm": 233.01123046875,
|
|
"rewards/chosen": 0.5972605275583791,
|
|
"rewards/margins": 0.9716978745433373,
|
|
"rewards/rejected": -0.37443734698495823,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 8.343278884887695,
|
|
"kl": 12.181922912597656,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33700287.37864078,
|
|
"logits/rejected": -34702329.81268882,
|
|
"logps/chosen": -461.6702063106796,
|
|
"logps/rejected": -376.6431032854985,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.1118314266204834,
|
|
"metrics/advantage_var": 344.9362487792969,
|
|
"regularization/mmd": 1.0864795446395874,
|
|
"regularization/rkhs_norm": 209.3409423828125,
|
|
"rewards/chosen": 0.5032996514082727,
|
|
"rewards/margins": 0.8964650784404669,
|
|
"rewards/rejected": -0.3931654270321941,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 9.558717727661133,
|
|
"kl": 11.3455228805542,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34344430.03508772,
|
|
"logits/rejected": -36126331.70469799,
|
|
"logps/chosen": -456.03682383040933,
|
|
"logps/rejected": -363.75760276845637,
|
|
"loss": 0.5249,
|
|
"loss/base_kto": 3.1258723735809326,
|
|
"metrics/advantage_var": 312.35992431640625,
|
|
"regularization/mmd": 1.0734361410140991,
|
|
"regularization/rkhs_norm": 206.8277587890625,
|
|
"rewards/chosen": 0.5453165065475375,
|
|
"rewards/margins": 0.8938541820338526,
|
|
"rewards/rejected": -0.34853767548631504,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.555154800415039,
|
|
"kl": 16.892311096191406,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35235413.19547658,
|
|
"logits/rejected": -35423521.69440242,
|
|
"logps/chosen": -476.1195476575121,
|
|
"logps/rejected": -373.17310419818455,
|
|
"loss": 0.5232,
|
|
"loss/base_kto": 3.075972557067871,
|
|
"metrics/advantage_var": 347.23553466796875,
|
|
"regularization/mmd": 1.1093342304229736,
|
|
"regularization/rkhs_norm": 213.74453735351562,
|
|
"rewards/chosen": 0.5682295150633583,
|
|
"rewards/margins": 0.8934559222515084,
|
|
"rewards/rejected": -0.32522640718815005,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 13.49815559387207,
|
|
"kl": 14.067483901977539,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34997816.88888889,
|
|
"logits/rejected": -36328561.412520066,
|
|
"logps/chosen": -481.9634703196347,
|
|
"logps/rejected": -379.16470204654894,
|
|
"loss": 0.5331,
|
|
"loss/base_kto": 3.0894711017608643,
|
|
"metrics/advantage_var": 401.41070556640625,
|
|
"regularization/mmd": 1.1757266521453857,
|
|
"regularization/rkhs_norm": 226.5369110107422,
|
|
"rewards/chosen": 0.560076987906678,
|
|
"rewards/margins": 0.9336913037172719,
|
|
"rewards/rejected": -0.3736143158105939,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 14.797484397888184,
|
|
"kl": 14.350302696228027,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -36001405.267651886,
|
|
"logits/rejected": -36866606.54545455,
|
|
"logps/chosen": -478.26472701149424,
|
|
"logps/rejected": -379.9672596870343,
|
|
"loss": 0.5311,
|
|
"loss/base_kto": 3.098508834838867,
|
|
"metrics/advantage_var": 388.4842224121094,
|
|
"regularization/mmd": 1.15003502368927,
|
|
"regularization/rkhs_norm": 221.5867156982422,
|
|
"rewards/chosen": 0.5771887768274067,
|
|
"rewards/margins": 0.9156153923761445,
|
|
"rewards/rejected": -0.3384266155487379,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 8.583003044128418,
|
|
"kl": 11.744126319885254,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34785482.27160494,
|
|
"logits/rejected": -35954642.63291139,
|
|
"logps/chosen": -503.56037808641975,
|
|
"logps/rejected": -367.19931764240505,
|
|
"loss": 0.5307,
|
|
"loss/base_kto": 3.065383195877075,
|
|
"metrics/advantage_var": 400.19427490234375,
|
|
"regularization/mmd": 1.180153489112854,
|
|
"regularization/rkhs_norm": 227.38987731933594,
|
|
"rewards/chosen": 0.5370976954330633,
|
|
"rewards/margins": 0.9577066988139473,
|
|
"rewards/rejected": -0.4206090033808841,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 9.572432518005371,
|
|
"kl": 13.085634231567383,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34060697.6,
|
|
"logits/rejected": -36925001.37322835,
|
|
"logps/chosen": -482.0249031007752,
|
|
"logps/rejected": -365.8713090551181,
|
|
"loss": 0.5286,
|
|
"loss/base_kto": 3.038194179534912,
|
|
"metrics/advantage_var": 450.6354675292969,
|
|
"regularization/mmd": 1.1905380487442017,
|
|
"regularization/rkhs_norm": 229.39077758789062,
|
|
"rewards/chosen": 0.5976823200551115,
|
|
"rewards/margins": 1.0057066168532167,
|
|
"rewards/rejected": -0.4080242967981053,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 8.293378829956055,
|
|
"kl": 12.306334495544434,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -33829552.98765432,
|
|
"logits/rejected": -35876339.037974685,
|
|
"logps/chosen": -473.9470486111111,
|
|
"logps/rejected": -360.09342859968353,
|
|
"loss": 0.5253,
|
|
"loss/base_kto": 3.0924527645111084,
|
|
"metrics/advantage_var": 365.6189880371094,
|
|
"regularization/mmd": 1.1100748777389526,
|
|
"regularization/rkhs_norm": 213.8872528076172,
|
|
"rewards/chosen": 0.5788733870894821,
|
|
"rewards/margins": 0.941765272388199,
|
|
"rewards/rejected": -0.36289188529871685,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 9.167913436889648,
|
|
"kl": 10.900675773620605,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35208733.538461536,
|
|
"logits/rejected": -36294346.92682927,
|
|
"logps/chosen": -478.72000200320514,
|
|
"logps/rejected": -382.6992663871951,
|
|
"loss": 0.5297,
|
|
"loss/base_kto": 3.080029010772705,
|
|
"metrics/advantage_var": 387.3747253417969,
|
|
"regularization/mmd": 1.1578792333602905,
|
|
"regularization/rkhs_norm": 223.0980987548828,
|
|
"rewards/chosen": 0.5555437528170072,
|
|
"rewards/margins": 0.9557336875242766,
|
|
"rewards/rejected": -0.40018993470726943,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.086071968078613,
|
|
"kl": 10.250889778137207,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34755010.81761006,
|
|
"logits/rejected": -36850360.44720497,
|
|
"logps/chosen": -499.359768081761,
|
|
"logps/rejected": -360.2353454968944,
|
|
"loss": 0.5276,
|
|
"loss/base_kto": 3.0793330669403076,
|
|
"metrics/advantage_var": 378.3411560058594,
|
|
"regularization/mmd": 1.1418160200119019,
|
|
"regularization/rkhs_norm": 220.00308227539062,
|
|
"rewards/chosen": 0.5264859949267885,
|
|
"rewards/margins": 0.9395983746761576,
|
|
"rewards/rejected": -0.41311237974936915,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 10.675963401794434,
|
|
"kl": 10.080180168151855,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34271557.178625956,
|
|
"logits/rejected": -34417416.6016,
|
|
"logps/chosen": -491.93396946564883,
|
|
"logps/rejected": -365.516775,
|
|
"loss": 0.5283,
|
|
"loss/base_kto": 3.093646764755249,
|
|
"metrics/advantage_var": 370.02587890625,
|
|
"regularization/mmd": 1.1326717138290405,
|
|
"regularization/rkhs_norm": 218.2411651611328,
|
|
"rewards/chosen": 0.53616943359375,
|
|
"rewards/margins": 0.9338271484375,
|
|
"rewards/rejected": -0.39765771484375,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 10.079834938049316,
|
|
"kl": 10.984450340270996,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34487719.67022901,
|
|
"logits/rejected": -35446177.792,
|
|
"logps/chosen": -481.07776717557255,
|
|
"logps/rejected": -361.6391,
|
|
"loss": 0.5229,
|
|
"loss/base_kto": 3.10184907913208,
|
|
"metrics/advantage_var": 329.1188659667969,
|
|
"regularization/mmd": 1.0813477039337158,
|
|
"regularization/rkhs_norm": 208.35215759277344,
|
|
"rewards/chosen": 0.5546003734792462,
|
|
"rewards/margins": 0.9016446850026837,
|
|
"rewards/rejected": -0.3470443115234375,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.892107009887695,
|
|
"kl": 11.589664459228516,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35282340.18237082,
|
|
"logits/rejected": -36262250.18649518,
|
|
"logps/chosen": -447.93531534954406,
|
|
"logps/rejected": -357.67041800643085,
|
|
"loss": 0.5278,
|
|
"loss/base_kto": 3.116950035095215,
|
|
"metrics/advantage_var": 351.3929443359375,
|
|
"regularization/mmd": 1.105540156364441,
|
|
"regularization/rkhs_norm": 213.01353454589844,
|
|
"rewards/chosen": 0.5573816270451415,
|
|
"rewards/margins": 0.9172011926004319,
|
|
"rewards/rejected": -0.3598195655552904,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 12.195263862609863,
|
|
"kl": 13.642374038696289,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -35447195.44227886,
|
|
"logits/rejected": -38002390.65579119,
|
|
"logps/chosen": -477.7605416041979,
|
|
"logps/rejected": -384.575397634584,
|
|
"loss": 0.5267,
|
|
"loss/base_kto": 3.0857346057891846,
|
|
"metrics/advantage_var": 371.4261779785156,
|
|
"regularization/mmd": 1.1274933815002441,
|
|
"regularization/rkhs_norm": 217.24343872070312,
|
|
"rewards/chosen": 0.5979615630417213,
|
|
"rewards/margins": 0.9352603561188013,
|
|
"rewards/rejected": -0.33729879307707994,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 9.376483917236328,
|
|
"kl": 15.698315620422363,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35467858.381954886,
|
|
"logits/rejected": -35734383.141463414,
|
|
"logps/chosen": -448.4790883458647,
|
|
"logps/rejected": -394.01725101626016,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.108189105987549,
|
|
"metrics/advantage_var": 376.1659851074219,
|
|
"regularization/mmd": 1.1289169788360596,
|
|
"regularization/rkhs_norm": 217.51773071289062,
|
|
"rewards/chosen": 0.5993606452655075,
|
|
"rewards/margins": 0.9191926050597148,
|
|
"rewards/rejected": -0.3198319597942073,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 9.31547737121582,
|
|
"kl": 13.520224571228027,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34428061.786163524,
|
|
"logits/rejected": -35251276.322981365,
|
|
"logps/chosen": -487.68391312893084,
|
|
"logps/rejected": -390.0930221273292,
|
|
"loss": 0.5323,
|
|
"loss/base_kto": 3.0711276531219482,
|
|
"metrics/advantage_var": 403.23406982421875,
|
|
"regularization/mmd": 1.187195897102356,
|
|
"regularization/rkhs_norm": 228.7467803955078,
|
|
"rewards/chosen": 0.6404232768892492,
|
|
"rewards/margins": 0.9392683666298839,
|
|
"rewards/rejected": -0.29884508974063473,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 9.886720657348633,
|
|
"kl": 13.426684379577637,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -33179677.210776545,
|
|
"logits/rejected": -34135167.8027735,
|
|
"logps/chosen": -485.5435816164818,
|
|
"logps/rejected": -370.60925462249617,
|
|
"loss": 0.5274,
|
|
"loss/base_kto": 3.0824615955352783,
|
|
"metrics/advantage_var": 390.8512878417969,
|
|
"regularization/mmd": 1.136972188949585,
|
|
"regularization/rkhs_norm": 219.0697784423828,
|
|
"rewards/chosen": 0.570498362207186,
|
|
"rewards/margins": 0.9357927378692623,
|
|
"rewards/rejected": -0.3652943756620763,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 17.49335289001465,
|
|
"kl": 14.857207298278809,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -33880664.83280757,
|
|
"logits/rejected": -36222288.0495356,
|
|
"logps/chosen": -486.42291009463725,
|
|
"logps/rejected": -374.14601876934984,
|
|
"loss": 0.5259,
|
|
"loss/base_kto": 3.1070611476898193,
|
|
"metrics/advantage_var": 340.4523620605469,
|
|
"regularization/mmd": 1.1003831624984741,
|
|
"regularization/rkhs_norm": 212.0198974609375,
|
|
"rewards/chosen": 0.570823789771047,
|
|
"rewards/margins": 0.8934752306542464,
|
|
"rewards/rejected": -0.32265144088319947,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 9.30859661102295,
|
|
"kl": 12.42811107635498,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34244507.17538462,
|
|
"logits/rejected": -35936613.58730159,
|
|
"logps/chosen": -472.44153846153847,
|
|
"logps/rejected": -379.248189484127,
|
|
"loss": 0.5226,
|
|
"loss/base_kto": 3.065871000289917,
|
|
"metrics/advantage_var": 350.88531494140625,
|
|
"regularization/mmd": 1.1149311065673828,
|
|
"regularization/rkhs_norm": 214.8229522705078,
|
|
"rewards/chosen": 0.5845942570612981,
|
|
"rewards/margins": 0.9485169090485253,
|
|
"rewards/rejected": -0.3639226519872272,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 10.817000389099121,
|
|
"kl": 15.556986808776855,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -33087391.427642275,
|
|
"logits/rejected": -35793348.715789475,
|
|
"logps/chosen": -485.847256097561,
|
|
"logps/rejected": -366.6436090225564,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.076166868209839,
|
|
"metrics/advantage_var": 368.3682556152344,
|
|
"regularization/mmd": 1.1028566360473633,
|
|
"regularization/rkhs_norm": 212.4964599609375,
|
|
"rewards/chosen": 0.5766742985423018,
|
|
"rewards/margins": 0.9292531945126965,
|
|
"rewards/rejected": -0.3525788959703947,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.865309715270996,
|
|
"kl": 13.899508476257324,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35078144.0,
|
|
"logits/rejected": -36999473.15514593,
|
|
"logps/chosen": -464.60597178060414,
|
|
"logps/rejected": -379.12608006912444,
|
|
"loss": 0.5262,
|
|
"loss/base_kto": 3.1125473976135254,
|
|
"metrics/advantage_var": 352.5972595214844,
|
|
"regularization/mmd": 1.0970466136932373,
|
|
"regularization/rkhs_norm": 211.37698364257812,
|
|
"rewards/chosen": 0.5437903957639855,
|
|
"rewards/margins": 0.9147923586396095,
|
|
"rewards/rejected": -0.37100196287562404,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 15.151383399963379,
|
|
"kl": 14.086331367492676,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -36041168.12678289,
|
|
"logits/rejected": -36800209.06009245,
|
|
"logps/chosen": -508.65674524564184,
|
|
"logps/rejected": -352.3438944530046,
|
|
"loss": 0.5273,
|
|
"loss/base_kto": 3.064643621444702,
|
|
"metrics/advantage_var": 382.234619140625,
|
|
"regularization/mmd": 1.153996229171753,
|
|
"regularization/rkhs_norm": 222.34996032714844,
|
|
"rewards/chosen": 0.5957659012542096,
|
|
"rewards/margins": 0.9596543993341884,
|
|
"rewards/rejected": -0.3638884980799788,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 12.178041458129883,
|
|
"kl": 15.788302421569824,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -35979590.77409162,
|
|
"logits/rejected": -36445850.3122102,
|
|
"logps/chosen": -487.91039691943126,
|
|
"logps/rejected": -390.93416731066463,
|
|
"loss": 0.5238,
|
|
"loss/base_kto": 3.0518758296966553,
|
|
"metrics/advantage_var": 377.42388916015625,
|
|
"regularization/mmd": 1.1385143995285034,
|
|
"regularization/rkhs_norm": 219.366943359375,
|
|
"rewards/chosen": 0.5787471150523301,
|
|
"rewards/margins": 0.9418147420054551,
|
|
"rewards/rejected": -0.363067626953125,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_loss": 0.5585662964506265,
|
|
"train_runtime": 11062.9611,
|
|
"train_samples_per_second": 13.398,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|