2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 22.61551284790039,
|
|
"kl": 17.726173400878906,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37893408.429210134,
|
|
"logits/rejected": -37338174.2134647,
|
|
"logps/chosen": -463.83597242921013,
|
|
"logps/rejected": -394.84208230706076,
|
|
"loss": 0.6117,
|
|
"loss/base_kto": 3.895165205001831,
|
|
"metrics/advantage_var": 223.74169921875,
|
|
"regularization/lipschitz_norm": 1031.2908935546875,
|
|
"regularization/w1": 0.9982895851135254,
|
|
"rewards/chosen": 0.29134313182574983,
|
|
"rewards/margins": 0.08558140555910498,
|
|
"rewards/rejected": 0.20576172626664485,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 29.268369674682617,
|
|
"kl": 7.626328468322754,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36383921.94098361,
|
|
"logits/rejected": -37250466.77014925,
|
|
"logps/chosen": -483.99006147540985,
|
|
"logps/rejected": -372.2102145522388,
|
|
"loss": 0.5907,
|
|
"loss/base_kto": 3.900822401046753,
|
|
"metrics/advantage_var": 168.17161560058594,
|
|
"regularization/lipschitz_norm": 852.3118286132812,
|
|
"regularization/w1": 0.825037956237793,
|
|
"rewards/chosen": 0.05755308807873335,
|
|
"rewards/margins": 0.09396182844975422,
|
|
"rewards/rejected": -0.03640874037102087,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 18.46786117553711,
|
|
"kl": 1.1394057273864746,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36016378.31111111,
|
|
"logits/rejected": -37364632.024615385,
|
|
"logps/chosen": -483.00376984126984,
|
|
"logps/rejected": -347.98759615384614,
|
|
"loss": 0.6011,
|
|
"loss/base_kto": 3.8704826831817627,
|
|
"metrics/advantage_var": 204.76536560058594,
|
|
"regularization/lipschitz_norm": 969.6908569335938,
|
|
"regularization/w1": 0.9386608004570007,
|
|
"rewards/chosen": -0.13712601434616817,
|
|
"rewards/margins": 0.12312108413640394,
|
|
"rewards/rejected": -0.2602470984825721,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 26.17717170715332,
|
|
"kl": 6.6596598625183105,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35716534.85714286,
|
|
"logits/rejected": -38076664.91076923,
|
|
"logps/chosen": -491.4034226190476,
|
|
"logps/rejected": -381.05134615384617,
|
|
"loss": 0.6097,
|
|
"loss/base_kto": 3.9487831592559814,
|
|
"metrics/advantage_var": 205.52835083007812,
|
|
"regularization/lipschitz_norm": 959.7703247070312,
|
|
"regularization/w1": 0.9290577173233032,
|
|
"rewards/chosen": -0.05661453973679315,
|
|
"rewards/margins": 0.0005846625806647798,
|
|
"rewards/rejected": -0.05719920231745793,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 24.305004119873047,
|
|
"kl": 0.8844783902168274,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35351495.380726695,
|
|
"logits/rejected": -35581217.63214838,
|
|
"logps/chosen": -499.3340244865719,
|
|
"logps/rejected": -365.4772024729521,
|
|
"loss": 0.5984,
|
|
"loss/base_kto": 3.8583943843841553,
|
|
"metrics/advantage_var": 189.59117126464844,
|
|
"regularization/lipschitz_norm": 959.7283325195312,
|
|
"regularization/w1": 0.9290170669555664,
|
|
"rewards/chosen": -0.14001726388554625,
|
|
"rewards/margins": 0.14358763658658474,
|
|
"rewards/rejected": -0.283604900472131,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 25.851768493652344,
|
|
"kl": 10.99168586730957,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35168529.878157504,
|
|
"logits/rejected": -37160055.77594728,
|
|
"logps/chosen": -483.59193907875186,
|
|
"logps/rejected": -377.68441103789127,
|
|
"loss": 0.5913,
|
|
"loss/base_kto": 3.818002462387085,
|
|
"metrics/advantage_var": 194.14869689941406,
|
|
"regularization/lipschitz_norm": 942.8248291015625,
|
|
"regularization/w1": 0.9126545190811157,
|
|
"rewards/chosen": 0.2000261507983725,
|
|
"rewards/margins": 0.16513429011297684,
|
|
"rewards/rejected": 0.03489186068539565,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 18.723735809326172,
|
|
"kl": 14.190146446228027,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -34976542.591194965,
|
|
"logits/rejected": -36276980.86956522,
|
|
"logps/chosen": -488.0369988207547,
|
|
"logps/rejected": -405.663189052795,
|
|
"loss": 0.5926,
|
|
"loss/base_kto": 3.81254506111145,
|
|
"metrics/advantage_var": 205.521728515625,
|
|
"regularization/lipschitz_norm": 959.1095581054688,
|
|
"regularization/w1": 0.928417980670929,
|
|
"rewards/chosen": 0.2530964065647725,
|
|
"rewards/margins": 0.1688930181848048,
|
|
"rewards/rejected": 0.08420338837996773,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 18.48126983642578,
|
|
"kl": 9.433762550354004,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35836909.15337423,
|
|
"logits/rejected": -37283781.299363054,
|
|
"logps/chosen": -460.209643404908,
|
|
"logps/rejected": -385.0283638535032,
|
|
"loss": 0.5933,
|
|
"loss/base_kto": 3.8530025482177734,
|
|
"metrics/advantage_var": 190.32125854492188,
|
|
"regularization/lipschitz_norm": 923.2146606445312,
|
|
"regularization/w1": 0.8936716318130493,
|
|
"rewards/chosen": 0.14656586442256997,
|
|
"rewards/margins": 0.1347334965802055,
|
|
"rewards/rejected": 0.011832367842364463,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 23.23832893371582,
|
|
"kl": 10.999950408935547,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35727274.392282955,
|
|
"logits/rejected": -37129882.0668693,
|
|
"logps/chosen": -498.3453074758842,
|
|
"logps/rejected": -370.99472834346506,
|
|
"loss": 0.593,
|
|
"loss/base_kto": 3.8195252418518066,
|
|
"metrics/advantage_var": 206.8389434814453,
|
|
"regularization/lipschitz_norm": 955.2222900390625,
|
|
"regularization/w1": 0.9246550798416138,
|
|
"rewards/chosen": 0.1618742866148136,
|
|
"rewards/margins": 0.16023750031336598,
|
|
"rewards/rejected": 0.0016367863014476278,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 20.174917221069336,
|
|
"kl": 10.249463081359863,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36790023.85276074,
|
|
"logits/rejected": -36648183.847133756,
|
|
"logps/chosen": -476.25532016871165,
|
|
"logps/rejected": -383.9594944267516,
|
|
"loss": 0.5926,
|
|
"loss/base_kto": 3.8088600635528564,
|
|
"metrics/advantage_var": 214.56057739257812,
|
|
"regularization/lipschitz_norm": 962.4050903320312,
|
|
"regularization/w1": 0.9316080212593079,
|
|
"rewards/chosen": 0.14388895327328172,
|
|
"rewards/margins": 0.1813243694766773,
|
|
"rewards/rejected": -0.03743541620339558,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 22.677093505859375,
|
|
"kl": 11.488913536071777,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36652774.1952,
|
|
"logits/rejected": -35998462.04580153,
|
|
"logps/chosen": -489.1762,
|
|
"logps/rejected": -355.66991889312976,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.8311614990234375,
|
|
"metrics/advantage_var": 200.62957763671875,
|
|
"regularization/lipschitz_norm": 952.9503784179688,
|
|
"regularization/w1": 0.9224559664726257,
|
|
"rewards/chosen": 0.1296946044921875,
|
|
"rewards/margins": 0.1469469397857899,
|
|
"rewards/rejected": -0.017252335293602396,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 25.6236572265625,
|
|
"kl": 16.71211814880371,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35016283.00463679,
|
|
"logits/rejected": -36606003.766192734,
|
|
"logps/chosen": -465.0539992272025,
|
|
"logps/rejected": -363.51128060821486,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.722736358642578,
|
|
"metrics/advantage_var": 248.9115447998047,
|
|
"regularization/lipschitz_norm": 993.5855712890625,
|
|
"regularization/w1": 0.9617907404899597,
|
|
"rewards/chosen": 0.320264577497102,
|
|
"rewards/margins": 0.2624493150054329,
|
|
"rewards/rejected": 0.057815262491669135,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.011987686157227,
|
|
"kl": 5.227573871612549,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36314480.38034865,
|
|
"logits/rejected": -38333952.78890601,
|
|
"logps/chosen": -496.1202456418383,
|
|
"logps/rejected": -386.7652157164869,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.800288438796997,
|
|
"metrics/advantage_var": 247.61289978027344,
|
|
"regularization/lipschitz_norm": 1077.5584716796875,
|
|
"regularization/w1": 1.0430763959884644,
|
|
"rewards/chosen": 0.020326623448101353,
|
|
"rewards/margins": 0.19650560119105326,
|
|
"rewards/rejected": -0.1761789777429519,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 17.925739288330078,
|
|
"kl": 5.149405002593994,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -35811164.94267516,
|
|
"logits/rejected": -36887809.57055215,
|
|
"logps/chosen": -484.67257165605093,
|
|
"logps/rejected": -371.10280866564415,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.77862548828125,
|
|
"metrics/advantage_var": 242.25364685058594,
|
|
"regularization/lipschitz_norm": 1019.4447631835938,
|
|
"regularization/w1": 0.9868223071098328,
|
|
"rewards/chosen": 0.015709421437257415,
|
|
"rewards/margins": 0.20944999702873676,
|
|
"rewards/rejected": -0.19374057559147934,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 22.183860778808594,
|
|
"kl": 8.57587718963623,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37178321.52738654,
|
|
"logits/rejected": -37794692.99219969,
|
|
"logps/chosen": -485.09448356807513,
|
|
"logps/rejected": -376.52296216848674,
|
|
"loss": 0.5897,
|
|
"loss/base_kto": 3.7043817043304443,
|
|
"metrics/advantage_var": 265.66552734375,
|
|
"regularization/lipschitz_norm": 1046.708984375,
|
|
"regularization/w1": 1.0132144689559937,
|
|
"rewards/chosen": 0.1665053986980695,
|
|
"rewards/margins": 0.27850994510307026,
|
|
"rewards/rejected": -0.11200454640500074,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 17.32640838623047,
|
|
"kl": 6.009871006011963,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35340411.003003,
|
|
"logits/rejected": -36968414.64495114,
|
|
"logps/chosen": -485.8840559309309,
|
|
"logps/rejected": -349.19360749185665,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.793537139892578,
|
|
"metrics/advantage_var": 233.20884704589844,
|
|
"regularization/lipschitz_norm": 981.0484619140625,
|
|
"regularization/w1": 0.9496549963951111,
|
|
"rewards/chosen": 0.05680670752539649,
|
|
"rewards/margins": 0.21321438996285833,
|
|
"rewards/rejected": -0.15640768243746184,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.678268432617188,
|
|
"kl": 9.558260917663574,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35297317.01204819,
|
|
"logits/rejected": -36870852.15584416,
|
|
"logps/chosen": -477.7679781626506,
|
|
"logps/rejected": -387.1367948457792,
|
|
"loss": 0.5889,
|
|
"loss/base_kto": 3.772874116897583,
|
|
"metrics/advantage_var": 212.17727661132812,
|
|
"regularization/lipschitz_norm": 969.6173095703125,
|
|
"regularization/w1": 0.9385895133018494,
|
|
"rewards/chosen": 0.19018784488540097,
|
|
"rewards/margins": 0.20245667299256603,
|
|
"rewards/rejected": -0.012268828107165052,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 22.296146392822266,
|
|
"kl": 15.797822952270508,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34978442.003189795,
|
|
"logits/rejected": -36334581.0229709,
|
|
"logps/chosen": -499.66417464114835,
|
|
"logps/rejected": -354.1043501148545,
|
|
"loss": 0.591,
|
|
"loss/base_kto": 3.765805959701538,
|
|
"metrics/advantage_var": 213.3668975830078,
|
|
"regularization/lipschitz_norm": 994.2394409179688,
|
|
"regularization/w1": 0.9624237418174744,
|
|
"rewards/chosen": 0.2144070614658094,
|
|
"rewards/margins": 0.19706860519072236,
|
|
"rewards/rejected": 0.01733845627508704,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 24.641950607299805,
|
|
"kl": 7.255638122558594,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36362105.6512,
|
|
"logits/rejected": -38003601.00152672,
|
|
"logps/chosen": -495.8204,
|
|
"logps/rejected": -375.6777671755725,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.740105390548706,
|
|
"metrics/advantage_var": 270.93975830078125,
|
|
"regularization/lipschitz_norm": 1074.212158203125,
|
|
"regularization/w1": 1.0398372411727905,
|
|
"rewards/chosen": 0.06908189086914063,
|
|
"rewards/margins": 0.2576003393275137,
|
|
"rewards/rejected": -0.1885184484583731,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 17.612085342407227,
|
|
"kl": 7.0018157958984375,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35700540.65230769,
|
|
"logits/rejected": -37447728.76190476,
|
|
"logps/chosen": -490.6022115384615,
|
|
"logps/rejected": -384.22442956349204,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.771484136581421,
|
|
"metrics/advantage_var": 257.8808288574219,
|
|
"regularization/lipschitz_norm": 1063.3167724609375,
|
|
"regularization/w1": 1.0292905569076538,
|
|
"rewards/chosen": 0.1267694326547476,
|
|
"rewards/margins": 0.2220495137830648,
|
|
"rewards/rejected": -0.0952800811283172,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 19.3057918548584,
|
|
"kl": 14.914984703063965,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35668982.518518515,
|
|
"logits/rejected": -39240788.25316456,
|
|
"logps/chosen": -492.21387924382714,
|
|
"logps/rejected": -355.3556912579114,
|
|
"loss": 0.5822,
|
|
"loss/base_kto": 3.6853816509246826,
|
|
"metrics/advantage_var": 262.89837646484375,
|
|
"regularization/lipschitz_norm": 1004.1189575195312,
|
|
"regularization/w1": 0.9719871878623962,
|
|
"rewards/chosen": 0.2619024912516276,
|
|
"rewards/margins": 0.28822025065683615,
|
|
"rewards/rejected": -0.02631775940520854,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 23.008037567138672,
|
|
"kl": 11.956717491149902,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -34780769.1318328,
|
|
"logits/rejected": -36797866.40729483,
|
|
"logps/chosen": -477.988344051447,
|
|
"logps/rejected": -374.3081781914894,
|
|
"loss": 0.6018,
|
|
"loss/base_kto": 3.7611629962921143,
|
|
"metrics/advantage_var": 257.25885009765625,
|
|
"regularization/lipschitz_norm": 1088.156494140625,
|
|
"regularization/w1": 1.0533353090286255,
|
|
"rewards/chosen": 0.11665805436407255,
|
|
"rewards/margins": 0.23021567860122585,
|
|
"rewards/rejected": -0.1135576242371533,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 21.199129104614258,
|
|
"kl": 9.169313430786133,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35709703.75757576,
|
|
"logits/rejected": -36700483.716129035,
|
|
"logps/chosen": -460.24460227272726,
|
|
"logps/rejected": -381.5554435483871,
|
|
"loss": 0.598,
|
|
"loss/base_kto": 3.760751485824585,
|
|
"metrics/advantage_var": 256.7850036621094,
|
|
"regularization/lipschitz_norm": 1056.8564453125,
|
|
"regularization/w1": 1.023037075996399,
|
|
"rewards/chosen": 0.18825216582327178,
|
|
"rewards/margins": 0.23152568244747634,
|
|
"rewards/rejected": -0.043273516624204575,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 27.59272575378418,
|
|
"kl": 7.0042724609375,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37679747.94165316,
|
|
"logits/rejected": -37919384.132730015,
|
|
"logps/chosen": -501.1313816855754,
|
|
"logps/rejected": -365.96021870286575,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.720513105392456,
|
|
"metrics/advantage_var": 290.1617126464844,
|
|
"regularization/lipschitz_norm": 1113.8450927734375,
|
|
"regularization/w1": 1.0782021284103394,
|
|
"rewards/chosen": 0.09823385822908225,
|
|
"rewards/margins": 0.27393960829315533,
|
|
"rewards/rejected": -0.1757057500640731,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.752378463745117,
|
|
"kl": 10.725379943847656,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35477635.724349156,
|
|
"logits/rejected": -36208378.69218501,
|
|
"logps/chosen": -494.9874617151608,
|
|
"logps/rejected": -381.44734350079744,
|
|
"loss": 0.5858,
|
|
"loss/base_kto": 3.7046101093292236,
|
|
"metrics/advantage_var": 234.78515625,
|
|
"regularization/lipschitz_norm": 1014.1542358398438,
|
|
"regularization/w1": 0.9817013144493103,
|
|
"rewards/chosen": 0.1933942271958449,
|
|
"rewards/margins": 0.2569185859665287,
|
|
"rewards/rejected": -0.06352435877068381,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 23.219057083129883,
|
|
"kl": 9.880836486816406,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36577298.73170732,
|
|
"logits/rejected": -37500960.820512824,
|
|
"logps/chosen": -478.9034870426829,
|
|
"logps/rejected": -366.79422075320514,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.789264678955078,
|
|
"metrics/advantage_var": 215.0338592529297,
|
|
"regularization/lipschitz_norm": 981.8986206054688,
|
|
"regularization/w1": 0.9504777789115906,
|
|
"rewards/chosen": 0.09943557367092226,
|
|
"rewards/margins": 0.1600638838989277,
|
|
"rewards/rejected": -0.06062831022800543,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 23.475032806396484,
|
|
"kl": 17.236083984375,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37205993.4591195,
|
|
"logits/rejected": -38576764.02484472,
|
|
"logps/chosen": -482.5070754716981,
|
|
"logps/rejected": -377.8761888586956,
|
|
"loss": 0.5985,
|
|
"loss/base_kto": 3.7453103065490723,
|
|
"metrics/advantage_var": 252.90396118164062,
|
|
"regularization/lipschitz_norm": 1076.9658203125,
|
|
"regularization/w1": 1.0425028800964355,
|
|
"rewards/chosen": 0.296068659368551,
|
|
"rewards/margins": 0.20513520910803323,
|
|
"rewards/rejected": 0.09093345026051776,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 22.165006637573242,
|
|
"kl": 11.516542434692383,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35175617.20754717,
|
|
"logits/rejected": -37775474.48447205,
|
|
"logps/chosen": -473.7337853773585,
|
|
"logps/rejected": -359.1034064440994,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.7129135131835938,
|
|
"metrics/advantage_var": 256.4697265625,
|
|
"regularization/lipschitz_norm": 1052.5374755859375,
|
|
"regularization/w1": 1.0188562870025635,
|
|
"rewards/chosen": 0.25117048827357263,
|
|
"rewards/margins": 0.255254651844569,
|
|
"rewards/rejected": -0.004084163570996397,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.9310359954834,
|
|
"kl": 12.891018867492676,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35657514.53776435,
|
|
"logits/rejected": -36446983.45631068,
|
|
"logps/chosen": -482.76680513595164,
|
|
"logps/rejected": -364.8552791262136,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7332704067230225,
|
|
"metrics/advantage_var": 258.4113464355469,
|
|
"regularization/lipschitz_norm": 1090.2811279296875,
|
|
"regularization/w1": 1.0553919076919556,
|
|
"rewards/chosen": 0.243781421119713,
|
|
"rewards/margins": 0.25185478036533426,
|
|
"rewards/rejected": -0.00807335924562127,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 27.660295486450195,
|
|
"kl": 11.09241771697998,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36485962.71770335,
|
|
"logits/rejected": -36978882.45022971,
|
|
"logps/chosen": -482.4744816586922,
|
|
"logps/rejected": -372.57580398162327,
|
|
"loss": 0.5955,
|
|
"loss/base_kto": 3.7170052528381348,
|
|
"metrics/advantage_var": 254.47903442382812,
|
|
"regularization/lipschitz_norm": 1081.9442138671875,
|
|
"regularization/w1": 1.0473219156265259,
|
|
"rewards/chosen": 0.15547682718036657,
|
|
"rewards/margins": 0.25378128362538,
|
|
"rewards/rejected": -0.0983044564450134,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 23.2921085357666,
|
|
"kl": 14.014410972595215,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37297054.71175523,
|
|
"logits/rejected": -37293576.54628225,
|
|
"logps/chosen": -504.9441425120773,
|
|
"logps/rejected": -389.2247723823976,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.727022647857666,
|
|
"metrics/advantage_var": 236.0036163330078,
|
|
"regularization/lipschitz_norm": 1063.3231201171875,
|
|
"regularization/w1": 1.0292967557907104,
|
|
"rewards/chosen": 0.19683140065167068,
|
|
"rewards/margins": 0.24935243624692924,
|
|
"rewards/rejected": -0.05252103559525856,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 21.310455322265625,
|
|
"kl": 11.659140586853027,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35771240.540192924,
|
|
"logits/rejected": -36612967.4893617,
|
|
"logps/chosen": -479.94433279742765,
|
|
"logps/rejected": -383.04796732522794,
|
|
"loss": 0.5955,
|
|
"loss/base_kto": 3.7672271728515625,
|
|
"metrics/advantage_var": 243.11131286621094,
|
|
"regularization/lipschitz_norm": 1029.5413818359375,
|
|
"regularization/w1": 0.9965960383415222,
|
|
"rewards/chosen": 0.10298527168690967,
|
|
"rewards/margins": 0.20089206265504228,
|
|
"rewards/rejected": -0.0979067909681326,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 38.099945068359375,
|
|
"kl": 16.773916244506836,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36485869.26829268,
|
|
"logits/rejected": -36950294.974358976,
|
|
"logps/chosen": -503.9336890243902,
|
|
"logps/rejected": -381.6227213541667,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.734379529953003,
|
|
"metrics/advantage_var": 271.9481506347656,
|
|
"regularization/lipschitz_norm": 1119.2958984375,
|
|
"regularization/w1": 1.083478331565857,
|
|
"rewards/chosen": 0.28929428937958507,
|
|
"rewards/margins": 0.24367292177833713,
|
|
"rewards/rejected": 0.045621367601247936,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 18.359342575073242,
|
|
"kl": 18.409404754638672,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35800295.541401275,
|
|
"logits/rejected": -35299252.61349693,
|
|
"logps/chosen": -469.1358976910828,
|
|
"logps/rejected": -364.7176715874233,
|
|
"loss": 0.594,
|
|
"loss/base_kto": 3.6930909156799316,
|
|
"metrics/advantage_var": 275.6893615722656,
|
|
"regularization/lipschitz_norm": 1094.0638427734375,
|
|
"regularization/w1": 1.0590537786483765,
|
|
"rewards/chosen": 0.3039243661673965,
|
|
"rewards/margins": 0.27000463975467603,
|
|
"rewards/rejected": 0.03391972641272047,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 37.54812240600586,
|
|
"kl": 10.203047752380371,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37182692.9305136,
|
|
"logits/rejected": -37239247.94822007,
|
|
"logps/chosen": -477.6900962990937,
|
|
"logps/rejected": -371.96875,
|
|
"loss": 0.603,
|
|
"loss/base_kto": 3.7724015712738037,
|
|
"metrics/advantage_var": 259.531494140625,
|
|
"regularization/lipschitz_norm": 1086.3216552734375,
|
|
"regularization/w1": 1.051559329032898,
|
|
"rewards/chosen": 0.09012801985726256,
|
|
"rewards/margins": 0.2202559606061501,
|
|
"rewards/rejected": -0.13012794074888753,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 19.995777130126953,
|
|
"kl": 19.587913513183594,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35426880.90140845,
|
|
"logits/rejected": -35140095.20124805,
|
|
"logps/chosen": -492.51085680751174,
|
|
"logps/rejected": -373.3675897035881,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.656334400177002,
|
|
"metrics/advantage_var": 249.142822265625,
|
|
"regularization/lipschitz_norm": 1083.2322998046875,
|
|
"regularization/w1": 1.048568844795227,
|
|
"rewards/chosen": 0.317853375406519,
|
|
"rewards/margins": 0.290600606828979,
|
|
"rewards/rejected": 0.027252768577539978,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 46.883968353271484,
|
|
"kl": 21.90428924560547,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35776368.50955414,
|
|
"logits/rejected": -36817436.26993865,
|
|
"logps/chosen": -474.05996218152865,
|
|
"logps/rejected": -358.19500095858893,
|
|
"loss": 0.5894,
|
|
"loss/base_kto": 3.702869415283203,
|
|
"metrics/advantage_var": 256.7891845703125,
|
|
"regularization/lipschitz_norm": 1045.6673583984375,
|
|
"regularization/w1": 1.0122060775756836,
|
|
"rewards/chosen": 0.3643581609057773,
|
|
"rewards/margins": 0.24737368237068091,
|
|
"rewards/rejected": 0.11698447853509635,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 22.3043155670166,
|
|
"kl": 13.030131340026855,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -34736035.34477379,
|
|
"logits/rejected": -36060722.47887324,
|
|
"logps/chosen": -508.2879290171607,
|
|
"logps/rejected": -371.79491881846633,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.744248628616333,
|
|
"metrics/advantage_var": 254.7359619140625,
|
|
"regularization/lipschitz_norm": 1069.6435546875,
|
|
"regularization/w1": 1.0354150533676147,
|
|
"rewards/chosen": 0.151040706545254,
|
|
"rewards/margins": 0.23194629251331933,
|
|
"rewards/rejected": -0.08090558596806534,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 26.595840454101562,
|
|
"kl": 11.584898948669434,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -35833795.294851795,
|
|
"logits/rejected": -36832845.96546311,
|
|
"logps/chosen": -489.06679017160684,
|
|
"logps/rejected": -362.74347527472526,
|
|
"loss": 0.6011,
|
|
"loss/base_kto": 3.5896096229553223,
|
|
"metrics/advantage_var": 393.7645568847656,
|
|
"regularization/lipschitz_norm": 1259.10400390625,
|
|
"regularization/w1": 1.2188128232955933,
|
|
"rewards/chosen": 0.23580975465581122,
|
|
"rewards/margins": 0.4212782996083475,
|
|
"rewards/rejected": -0.1854685449525363,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 21.800310134887695,
|
|
"kl": 26.010726928710938,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -34446830.67076923,
|
|
"logits/rejected": -35252067.961904764,
|
|
"logps/chosen": -483.19057692307695,
|
|
"logps/rejected": -381.0619543650794,
|
|
"loss": 0.6209,
|
|
"loss/base_kto": 3.236591100692749,
|
|
"metrics/advantage_var": 728.69482421875,
|
|
"regularization/lipschitz_norm": 1787.618408203125,
|
|
"regularization/w1": 1.730414628982544,
|
|
"rewards/chosen": 0.7645455228365384,
|
|
"rewards/margins": 0.8732299536401098,
|
|
"rewards/rejected": -0.10868443080357143,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 16.8453426361084,
|
|
"kl": 17.86311149597168,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34530329.28395062,
|
|
"logits/rejected": -35855583.594936706,
|
|
"logps/chosen": -495.580680941358,
|
|
"logps/rejected": -399.43225870253167,
|
|
"loss": 0.62,
|
|
"loss/base_kto": 3.245777130126953,
|
|
"metrics/advantage_var": 709.994384765625,
|
|
"regularization/lipschitz_norm": 1771.098876953125,
|
|
"regularization/w1": 1.7144235372543335,
|
|
"rewards/chosen": 0.5647619741934317,
|
|
"rewards/margins": 0.8513128581988213,
|
|
"rewards/rejected": -0.28655088400538964,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 23.292949676513672,
|
|
"kl": 19.082748413085938,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -34898210.215780996,
|
|
"logits/rejected": -35900436.20030349,
|
|
"logps/chosen": -490.05454911433173,
|
|
"logps/rejected": -393.5083459787557,
|
|
"loss": 0.5966,
|
|
"loss/base_kto": 3.2628426551818848,
|
|
"metrics/advantage_var": 595.5111694335938,
|
|
"regularization/lipschitz_norm": 1559.912353515625,
|
|
"regularization/w1": 1.5099951028823853,
|
|
"rewards/chosen": 0.5989260957628825,
|
|
"rewards/margins": 0.7875744643552098,
|
|
"rewards/rejected": -0.1886483685923274,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 21.66841697692871,
|
|
"kl": 6.183696269989014,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34671666.7904,
|
|
"logits/rejected": -35740301.483969465,
|
|
"logps/chosen": -487.8895,
|
|
"logps/rejected": -390.0688454198473,
|
|
"loss": 0.6208,
|
|
"loss/base_kto": 3.3012702465057373,
|
|
"metrics/advantage_var": 663.9572143554688,
|
|
"regularization/lipschitz_norm": 1720.2760009765625,
|
|
"regularization/w1": 1.6652271747589111,
|
|
"rewards/chosen": 0.264857421875,
|
|
"rewards/margins": 0.8203466722328245,
|
|
"rewards/rejected": -0.5554892503578245,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 15.83867359161377,
|
|
"kl": 7.2326979637146,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35209350.821316615,
|
|
"logits/rejected": -36016210.940809965,
|
|
"logps/chosen": -472.8581504702194,
|
|
"logps/rejected": -364.3094820872274,
|
|
"loss": 0.607,
|
|
"loss/base_kto": 3.3075034618377686,
|
|
"metrics/advantage_var": 630.6771850585938,
|
|
"regularization/lipschitz_norm": 1599.310302734375,
|
|
"regularization/w1": 1.548132300376892,
|
|
"rewards/chosen": 0.2560465941234816,
|
|
"rewards/margins": 0.7796168598488757,
|
|
"rewards/rejected": -0.5235702657253942,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 20.087535858154297,
|
|
"kl": 10.071452140808105,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34956357.316923074,
|
|
"logits/rejected": -36206019.86031746,
|
|
"logps/chosen": -474.38697115384616,
|
|
"logps/rejected": -383.57896825396824,
|
|
"loss": 0.5899,
|
|
"loss/base_kto": 3.325768232345581,
|
|
"metrics/advantage_var": 507.9310607910156,
|
|
"regularization/lipschitz_norm": 1439.6236572265625,
|
|
"regularization/w1": 1.3935555219650269,
|
|
"rewards/chosen": 0.3780939659705529,
|
|
"rewards/margins": 0.7193564684894641,
|
|
"rewards/rejected": -0.3412625025189112,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 13.59377670288086,
|
|
"kl": 16.066341400146484,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35124335.68049155,
|
|
"logits/rejected": -36426629.901430845,
|
|
"logps/chosen": -488.8565668202765,
|
|
"logps/rejected": -355.8337639109698,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.2414710521698,
|
|
"metrics/advantage_var": 649.274169921875,
|
|
"regularization/lipschitz_norm": 1591.873046875,
|
|
"regularization/w1": 1.5409330129623413,
|
|
"rewards/chosen": 0.5456774546070948,
|
|
"rewards/margins": 0.8671934339797349,
|
|
"rewards/rejected": -0.3215159793726401,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.43739891052246,
|
|
"kl": 11.587209701538086,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -33973825.933227345,
|
|
"logits/rejected": -34953722.49462365,
|
|
"logps/chosen": -473.96114864864865,
|
|
"logps/rejected": -347.6049347158218,
|
|
"loss": 0.5924,
|
|
"loss/base_kto": 3.3000848293304443,
|
|
"metrics/advantage_var": 534.9948120117188,
|
|
"regularization/lipschitz_norm": 1486.8486328125,
|
|
"regularization/w1": 1.4392694234848022,
|
|
"rewards/chosen": 0.4382156347811506,
|
|
"rewards/margins": 0.7501723187589252,
|
|
"rewards/rejected": -0.3119566839777746,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 14.55308723449707,
|
|
"kl": 7.113320827484131,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34393926.4,
|
|
"logits/rejected": -35935897.6,
|
|
"logps/chosen": -437.600537109375,
|
|
"logps/rejected": -377.1755615234375,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.3974227905273438,
|
|
"metrics/advantage_var": 510.9998474121094,
|
|
"regularization/lipschitz_norm": 1390.6151123046875,
|
|
"regularization/w1": 1.3461154699325562,
|
|
"rewards/chosen": 0.20054919719696046,
|
|
"rewards/margins": 0.6592526197433473,
|
|
"rewards/rejected": -0.45870342254638674,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 20.532312393188477,
|
|
"kl": 11.9752836227417,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -33429356.070626,
|
|
"logits/rejected": -36452704.243531205,
|
|
"logps/chosen": -512.6451143659712,
|
|
"logps/rejected": -390.00085616438355,
|
|
"loss": 0.625,
|
|
"loss/base_kto": 3.2616021633148193,
|
|
"metrics/advantage_var": 725.3668212890625,
|
|
"regularization/lipschitz_norm": 1795.5465087890625,
|
|
"regularization/w1": 1.7380889654159546,
|
|
"rewards/chosen": 0.4411593662219101,
|
|
"rewards/margins": 0.8262716699804122,
|
|
"rewards/rejected": -0.3851123037585022,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 20.995637893676758,
|
|
"kl": 12.625959396362305,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -36112063.18987342,
|
|
"logits/rejected": -34620156.83950617,
|
|
"logps/chosen": -468.85957278481015,
|
|
"logps/rejected": -375.61188271604937,
|
|
"loss": 0.6247,
|
|
"loss/base_kto": 3.288928985595703,
|
|
"metrics/advantage_var": 1292.3441162109375,
|
|
"regularization/lipschitz_norm": 1765.2825927734375,
|
|
"regularization/w1": 1.7087936401367188,
|
|
"rewards/chosen": 0.4622271573996242,
|
|
"rewards/margins": 0.8298365582076995,
|
|
"rewards/rejected": -0.3676094008080753,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 24.140911102294922,
|
|
"kl": 18.27957534790039,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34437724.175491676,
|
|
"logits/rejected": -36863358.13893376,
|
|
"logps/chosen": -471.59351361573374,
|
|
"logps/rejected": -376.7125908723748,
|
|
"loss": 0.6149,
|
|
"loss/base_kto": 3.3231351375579834,
|
|
"metrics/advantage_var": 665.4700317382812,
|
|
"regularization/lipschitz_norm": 1648.6927490234375,
|
|
"regularization/w1": 1.595934510231018,
|
|
"rewards/chosen": 0.5793022121135117,
|
|
"rewards/margins": 0.7244230701041279,
|
|
"rewards/rejected": -0.14512085799061616,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 19.79603385925293,
|
|
"kl": 26.32721519470215,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34581654.18666667,
|
|
"logits/rejected": -36698975.20661157,
|
|
"logps/chosen": -482.1162962962963,
|
|
"logps/rejected": -359.8138688016529,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.294232130050659,
|
|
"metrics/advantage_var": 626.8693237304688,
|
|
"regularization/lipschitz_norm": 1576.6903076171875,
|
|
"regularization/w1": 1.5262361764907837,
|
|
"rewards/chosen": 0.7386127387152778,
|
|
"rewards/margins": 0.7364574899314411,
|
|
"rewards/rejected": 0.00215524878383668,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 14.683740615844727,
|
|
"kl": 21.47702980041504,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35653150.300455235,
|
|
"logits/rejected": -37041433.97101449,
|
|
"logps/chosen": -484.8400986342944,
|
|
"logps/rejected": -381.7333685587762,
|
|
"loss": 0.6125,
|
|
"loss/base_kto": 3.2820115089416504,
|
|
"metrics/advantage_var": 797.7058715820312,
|
|
"regularization/lipschitz_norm": 1671.8375244140625,
|
|
"regularization/w1": 1.6183385848999023,
|
|
"rewards/chosen": 0.5844798804414834,
|
|
"rewards/margins": 0.7289509675670558,
|
|
"rewards/rejected": -0.1444710871255724,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 17.051183700561523,
|
|
"kl": 9.099251747131348,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35140609.57296467,
|
|
"logits/rejected": -37269692.84578697,
|
|
"logps/chosen": -499.5424347158218,
|
|
"logps/rejected": -369.0848569157393,
|
|
"loss": 0.6076,
|
|
"loss/base_kto": 3.3209598064422607,
|
|
"metrics/advantage_var": 622.4037475585938,
|
|
"regularization/lipschitz_norm": 1590.6639404296875,
|
|
"regularization/w1": 1.5397626161575317,
|
|
"rewards/chosen": 0.3732815462689612,
|
|
"rewards/margins": 0.7491195182094794,
|
|
"rewards/rejected": -0.37583797194051816,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 19.764598846435547,
|
|
"kl": 11.906905174255371,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34825108.54320987,
|
|
"logits/rejected": -35921884.35443038,
|
|
"logps/chosen": -479.0851658950617,
|
|
"logps/rejected": -358.64240506329116,
|
|
"loss": 0.594,
|
|
"loss/base_kto": 3.2469890117645264,
|
|
"metrics/advantage_var": 569.92919921875,
|
|
"regularization/lipschitz_norm": 1555.1317138671875,
|
|
"regularization/w1": 1.5053675174713135,
|
|
"rewards/chosen": 0.48613465862509647,
|
|
"rewards/margins": 0.8398204963828348,
|
|
"rewards/rejected": -0.3536858377577383,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 24.98237419128418,
|
|
"kl": 15.6842679977417,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35456782.86970173,
|
|
"logits/rejected": -36521948.96423017,
|
|
"logps/chosen": -471.10105965463106,
|
|
"logps/rejected": -370.63224144634523,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.241093397140503,
|
|
"metrics/advantage_var": 653.30615234375,
|
|
"regularization/lipschitz_norm": 1628.729736328125,
|
|
"regularization/w1": 1.5766104459762573,
|
|
"rewards/chosen": 0.49397677866218603,
|
|
"rewards/margins": 0.8182870576659405,
|
|
"rewards/rejected": -0.32431027900375436,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.70745849609375,
|
|
"kl": 15.941064834594727,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34274962.62910798,
|
|
"logits/rejected": -36485570.496099845,
|
|
"logps/chosen": -497.79367175273865,
|
|
"logps/rejected": -393.05465093603743,
|
|
"loss": 0.6131,
|
|
"loss/base_kto": 3.320526123046875,
|
|
"metrics/advantage_var": 663.6347045898438,
|
|
"regularization/lipschitz_norm": 1636.6031494140625,
|
|
"regularization/w1": 1.584231972694397,
|
|
"rewards/chosen": 0.5142061952880478,
|
|
"rewards/margins": 0.7003573754661143,
|
|
"rewards/rejected": -0.1861511801780665,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.632219314575195,
|
|
"kl": 13.42111873626709,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -32557049.756097563,
|
|
"logits/rejected": -36331864.615384616,
|
|
"logps/chosen": -475.3487995426829,
|
|
"logps/rejected": -369.18704927884613,
|
|
"loss": 0.587,
|
|
"loss/base_kto": 3.300513982772827,
|
|
"metrics/advantage_var": 532.9873657226562,
|
|
"regularization/lipschitz_norm": 1441.3602294921875,
|
|
"regularization/w1": 1.395236611366272,
|
|
"rewards/chosen": 0.4872467227098418,
|
|
"rewards/margins": 0.7677278166789423,
|
|
"rewards/rejected": -0.28048109396910054,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 20.784940719604492,
|
|
"kl": 21.042142868041992,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35007650.69158878,
|
|
"logits/rejected": -35474810.78369906,
|
|
"logps/chosen": -462.86156542056074,
|
|
"logps/rejected": -369.96017829153607,
|
|
"loss": 0.5948,
|
|
"loss/base_kto": 3.2664496898651123,
|
|
"metrics/advantage_var": 625.43603515625,
|
|
"regularization/lipschitz_norm": 1540.9881591796875,
|
|
"regularization/w1": 1.4916763305664062,
|
|
"rewards/chosen": 0.6386737764067367,
|
|
"rewards/margins": 0.8052515610257243,
|
|
"rewards/rejected": -0.16657778461898756,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 17.32353973388672,
|
|
"kl": 11.49404239654541,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35463141.44570503,
|
|
"logits/rejected": -36403119.686274506,
|
|
"logps/chosen": -472.8184764991896,
|
|
"logps/rejected": -352.13042043740575,
|
|
"loss": 0.5946,
|
|
"loss/base_kto": 3.3386070728302,
|
|
"metrics/advantage_var": 538.4461059570312,
|
|
"regularization/lipschitz_norm": 1464.8402099609375,
|
|
"regularization/w1": 1.417965292930603,
|
|
"rewards/chosen": 0.386352267025742,
|
|
"rewards/margins": 0.7313668823219807,
|
|
"rewards/rejected": -0.34501461529623867,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 19.162174224853516,
|
|
"kl": 17.63032341003418,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35381686.85714286,
|
|
"logits/rejected": -37131623.18769231,
|
|
"logps/chosen": -499.12628968253966,
|
|
"logps/rejected": -385.215,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.298352003097534,
|
|
"metrics/advantage_var": 577.2579956054688,
|
|
"regularization/lipschitz_norm": 1494.992431640625,
|
|
"regularization/w1": 1.4471524953842163,
|
|
"rewards/chosen": 0.4551373678540427,
|
|
"rewards/margins": 0.7502896740439465,
|
|
"rewards/rejected": -0.29515230618990385,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 17.638160705566406,
|
|
"kl": 15.187868118286133,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35187872.62745098,
|
|
"logits/rejected": -36121566.275449105,
|
|
"logps/chosen": -486.2753267973856,
|
|
"logps/rejected": -395.97913547904193,
|
|
"loss": 0.6129,
|
|
"loss/base_kto": 3.2702064514160156,
|
|
"metrics/advantage_var": 643.1571044921875,
|
|
"regularization/lipschitz_norm": 1686.6243896484375,
|
|
"regularization/w1": 1.6326522827148438,
|
|
"rewards/chosen": 0.42208243974673204,
|
|
"rewards/margins": 0.7818797061757881,
|
|
"rewards/rejected": -0.35979726642905596,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.01573944091797,
|
|
"kl": 9.487359046936035,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34373709.04075235,
|
|
"logits/rejected": -35589758.00623053,
|
|
"logps/chosen": -497.33140673981194,
|
|
"logps/rejected": -375.721281152648,
|
|
"loss": 0.6018,
|
|
"loss/base_kto": 3.2764053344726562,
|
|
"metrics/advantage_var": 580.426513671875,
|
|
"regularization/lipschitz_norm": 1588.6661376953125,
|
|
"regularization/w1": 1.537828803062439,
|
|
"rewards/chosen": 0.41636753381232855,
|
|
"rewards/margins": 0.7823655137578114,
|
|
"rewards/rejected": -0.3659979799454829,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 21.156766891479492,
|
|
"kl": 9.334273338317871,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34830602.85086342,
|
|
"logits/rejected": -36073401.928460345,
|
|
"logps/chosen": -499.9333791208791,
|
|
"logps/rejected": -372.42493681959564,
|
|
"loss": 0.6191,
|
|
"loss/base_kto": 3.3611772060394287,
|
|
"metrics/advantage_var": 629.4342041015625,
|
|
"regularization/lipschitz_norm": 1644.3922119140625,
|
|
"regularization/w1": 1.5917714834213257,
|
|
"rewards/chosen": 0.3352594061212225,
|
|
"rewards/margins": 0.6898744945054445,
|
|
"rewards/rejected": -0.3546150883842219,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 23.42906951904297,
|
|
"kl": 13.179911613464355,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -36929257.87654321,
|
|
"logits/rejected": -36407548.75949367,
|
|
"logps/chosen": -481.268856095679,
|
|
"logps/rejected": -380.07048556170884,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.306577444076538,
|
|
"metrics/advantage_var": 570.9453735351562,
|
|
"regularization/lipschitz_norm": 1553.357421875,
|
|
"regularization/w1": 1.5036499500274658,
|
|
"rewards/chosen": 0.46391852108048803,
|
|
"rewards/margins": 0.7183936438461378,
|
|
"rewards/rejected": -0.2544751227656497,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 17.261743545532227,
|
|
"kl": 11.59704875946045,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34884243.2,
|
|
"logits/rejected": -35944140.8,
|
|
"logps/chosen": -509.5953125,
|
|
"logps/rejected": -363.245556640625,
|
|
"loss": 0.6085,
|
|
"loss/base_kto": 3.3158957958221436,
|
|
"metrics/advantage_var": 582.8440551757812,
|
|
"regularization/lipschitz_norm": 1603.8192138671875,
|
|
"regularization/w1": 1.5524970293045044,
|
|
"rewards/chosen": 0.38999156951904296,
|
|
"rewards/margins": 0.732364273071289,
|
|
"rewards/rejected": -0.3423727035522461,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 21.07180404663086,
|
|
"kl": 11.0244140625,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33544349.02446982,
|
|
"logits/rejected": -34929822.12893553,
|
|
"logps/chosen": -464.3866231647635,
|
|
"logps/rejected": -372.451813155922,
|
|
"loss": 0.6008,
|
|
"loss/base_kto": 3.3070545196533203,
|
|
"metrics/advantage_var": 546.11376953125,
|
|
"regularization/lipschitz_norm": 1548.9422607421875,
|
|
"regularization/w1": 1.4993761777877808,
|
|
"rewards/chosen": 0.33109727769263614,
|
|
"rewards/margins": 0.71784746034057,
|
|
"rewards/rejected": -0.38675018264793387,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 19.116294860839844,
|
|
"kl": 9.207300186157227,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34389936.49689441,
|
|
"logits/rejected": -35984287.39622641,
|
|
"logps/chosen": -463.06327639751555,
|
|
"logps/rejected": -408.23786360062894,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.3536152839660645,
|
|
"metrics/advantage_var": 572.0338134765625,
|
|
"regularization/lipschitz_norm": 1504.552978515625,
|
|
"regularization/w1": 1.4564071893692017,
|
|
"rewards/chosen": 0.3255089943453392,
|
|
"rewards/margins": 0.6919748927751619,
|
|
"rewards/rejected": -0.36646589842982263,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 21.031543731689453,
|
|
"kl": 10.517437934875488,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35247339.80032206,
|
|
"logits/rejected": -36800151.502276175,
|
|
"logps/chosen": -486.52143719806764,
|
|
"logps/rejected": -394.4023852427921,
|
|
"loss": 0.6167,
|
|
"loss/base_kto": 3.3329575061798096,
|
|
"metrics/advantage_var": 612.3126831054688,
|
|
"regularization/lipschitz_norm": 1653.721923828125,
|
|
"regularization/w1": 1.6008027791976929,
|
|
"rewards/chosen": 0.3014136174642713,
|
|
"rewards/margins": 0.7257554473112222,
|
|
"rewards/rejected": -0.42434182984695085,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.131656646728516,
|
|
"kl": 6.698751926422119,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34359353.69014084,
|
|
"logits/rejected": -34682828.8798752,
|
|
"logps/chosen": -467.41070031298904,
|
|
"logps/rejected": -363.97235764430576,
|
|
"loss": 0.6061,
|
|
"loss/base_kto": 3.352490186691284,
|
|
"metrics/advantage_var": 574.9630737304688,
|
|
"regularization/lipschitz_norm": 1545.9027099609375,
|
|
"regularization/w1": 1.4964336156845093,
|
|
"rewards/chosen": 0.28778023637702954,
|
|
"rewards/margins": 0.7440857298077043,
|
|
"rewards/rejected": -0.4563054934306747,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 21.114540100097656,
|
|
"kl": 8.279240608215332,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34537630.55483871,
|
|
"logits/rejected": -36046376.339393936,
|
|
"logps/chosen": -488.0250504032258,
|
|
"logps/rejected": -374.4557765151515,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.2804977893829346,
|
|
"metrics/advantage_var": 591.7890014648438,
|
|
"regularization/lipschitz_norm": 1614.3330078125,
|
|
"regularization/w1": 1.5626744031906128,
|
|
"rewards/chosen": 0.31956624677104334,
|
|
"rewards/margins": 0.7858509813236001,
|
|
"rewards/rejected": -0.4662847345525568,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 19.38349723815918,
|
|
"kl": 11.979022026062012,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -36146015.61445783,
|
|
"logits/rejected": -35302725.81818182,
|
|
"logps/chosen": -488.31988893072287,
|
|
"logps/rejected": -385.71996753246754,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.2539162635803223,
|
|
"metrics/advantage_var": 611.529052734375,
|
|
"regularization/lipschitz_norm": 1586.0054931640625,
|
|
"regularization/w1": 1.5352532863616943,
|
|
"rewards/chosen": 0.4698529530720538,
|
|
"rewards/margins": 0.8464411799849114,
|
|
"rewards/rejected": -0.37658822691285765,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 20.861276626586914,
|
|
"kl": 12.396482467651367,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34633788.89514867,
|
|
"logits/rejected": -36115243.53198128,
|
|
"logps/chosen": -485.30545774647885,
|
|
"logps/rejected": -367.8154494929797,
|
|
"loss": 0.5911,
|
|
"loss/base_kto": 3.289091110229492,
|
|
"metrics/advantage_var": 562.3770141601562,
|
|
"regularization/lipschitz_norm": 1487.4207763671875,
|
|
"regularization/w1": 1.4398232698440552,
|
|
"rewards/chosen": 0.4437481278731416,
|
|
"rewards/margins": 0.7884234151287725,
|
|
"rewards/rejected": -0.34467528725563085,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 19.111433029174805,
|
|
"kl": 11.73603343963623,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36290983.294832826,
|
|
"logits/rejected": -36096991.073954985,
|
|
"logps/chosen": -479.78590425531917,
|
|
"logps/rejected": -377.9596563504823,
|
|
"loss": 0.6152,
|
|
"loss/base_kto": 3.2357006072998047,
|
|
"metrics/advantage_var": 659.8549194335938,
|
|
"regularization/lipschitz_norm": 1741.893798828125,
|
|
"regularization/w1": 1.6861534118652344,
|
|
"rewards/chosen": 0.4998804341695954,
|
|
"rewards/margins": 0.8565492805698165,
|
|
"rewards/rejected": -0.35666884640022106,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 18.78829002380371,
|
|
"kl": 9.772809982299805,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35879267.26530612,
|
|
"logits/rejected": -36516653.87283237,
|
|
"logps/chosen": -465.80963010204084,
|
|
"logps/rejected": -378.1799132947977,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.3241653442382812,
|
|
"metrics/advantage_var": 555.0695190429688,
|
|
"regularization/lipschitz_norm": 1576.09375,
|
|
"regularization/w1": 1.5256588459014893,
|
|
"rewards/chosen": 0.33075140609222203,
|
|
"rewards/margins": 0.7227623539646706,
|
|
"rewards/rejected": -0.3920109478724485,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 18.655101776123047,
|
|
"kl": 6.049205303192139,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35240254.43902439,
|
|
"logits/rejected": -36435193.43589743,
|
|
"logps/chosen": -498.1463414634146,
|
|
"logps/rejected": -359.73452524038464,
|
|
"loss": 0.605,
|
|
"loss/base_kto": 3.335129499435425,
|
|
"metrics/advantage_var": 560.3248291015625,
|
|
"regularization/lipschitz_norm": 1554.9830322265625,
|
|
"regularization/w1": 1.5052235126495361,
|
|
"rewards/chosen": 0.23430573068014005,
|
|
"rewards/margins": 0.7539112205576941,
|
|
"rewards/rejected": -0.5196054898775541,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 20.932680130004883,
|
|
"kl": 11.282088279724121,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34745622.72189349,
|
|
"logits/rejected": -36486503.41721854,
|
|
"logps/chosen": -485.5318047337278,
|
|
"logps/rejected": -370.7444639900662,
|
|
"loss": 0.6061,
|
|
"loss/base_kto": 3.3267104625701904,
|
|
"metrics/advantage_var": 577.7645874023438,
|
|
"regularization/lipschitz_norm": 1572.5972900390625,
|
|
"regularization/w1": 1.522274136543274,
|
|
"rewards/chosen": 0.44450044349805845,
|
|
"rewards/margins": 0.7331363037138077,
|
|
"rewards/rejected": -0.28863586021574916,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 15.270308494567871,
|
|
"kl": 17.181119918823242,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34989366.76616915,
|
|
"logits/rejected": -36915457.8962963,
|
|
"logps/chosen": -481.052964344942,
|
|
"logps/rejected": -360.38916666666665,
|
|
"loss": 0.5617,
|
|
"loss/base_kto": 3.1734278202056885,
|
|
"metrics/advantage_var": 480.20709228515625,
|
|
"regularization/lipschitz_norm": 1363.9676513671875,
|
|
"regularization/w1": 1.3203204870224,
|
|
"rewards/chosen": 0.5815272291699057,
|
|
"rewards/margins": 0.8491762905848363,
|
|
"rewards/rejected": -0.26764906141493056,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 20.278337478637695,
|
|
"kl": 8.590179443359375,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34879850.76972625,
|
|
"logits/rejected": -35662734.56752656,
|
|
"logps/chosen": -465.47735507246375,
|
|
"logps/rejected": -375.18138277693475,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.1985442638397217,
|
|
"metrics/advantage_var": 473.2780456542969,
|
|
"regularization/lipschitz_norm": 1394.8795166015625,
|
|
"regularization/w1": 1.3502434492111206,
|
|
"rewards/chosen": 0.420198204052813,
|
|
"rewards/margins": 0.849558670461434,
|
|
"rewards/rejected": -0.429360466408621,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 23.854448318481445,
|
|
"kl": 7.3163347244262695,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33588370.75159235,
|
|
"logits/rejected": -36218898.84662577,
|
|
"logps/chosen": -458.65898686305735,
|
|
"logps/rejected": -380.2924175613497,
|
|
"loss": 0.5549,
|
|
"loss/base_kto": 3.1283676624298096,
|
|
"metrics/advantage_var": 464.0574645996094,
|
|
"regularization/lipschitz_norm": 1354.0435791015625,
|
|
"regularization/w1": 1.3107141256332397,
|
|
"rewards/chosen": 0.4381056378601463,
|
|
"rewards/margins": 0.9256654654189796,
|
|
"rewards/rejected": -0.48755982755883337,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 12.578923225402832,
|
|
"kl": 8.453057289123535,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34610426.45201238,
|
|
"logits/rejected": -36772676.643533126,
|
|
"logps/chosen": -484.20646284829724,
|
|
"logps/rejected": -378.3252661671924,
|
|
"loss": 0.5595,
|
|
"loss/base_kto": 3.1872541904449463,
|
|
"metrics/advantage_var": 466.6294860839844,
|
|
"regularization/lipschitz_norm": 1331.5301513671875,
|
|
"regularization/w1": 1.2889211177825928,
|
|
"rewards/chosen": 0.4097718041009578,
|
|
"rewards/margins": 0.8629459980518155,
|
|
"rewards/rejected": -0.4531741939508577,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 15.493986129760742,
|
|
"kl": 11.95250415802002,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33701711.682119206,
|
|
"logits/rejected": -35296637.72781065,
|
|
"logps/chosen": -451.88907284768214,
|
|
"logps/rejected": -374.33089866863907,
|
|
"loss": 0.5547,
|
|
"loss/base_kto": 3.1352806091308594,
|
|
"metrics/advantage_var": 467.86309814453125,
|
|
"regularization/lipschitz_norm": 1345.5999755859375,
|
|
"regularization/w1": 1.3025407791137695,
|
|
"rewards/chosen": 0.4550409885431757,
|
|
"rewards/margins": 0.9068607752018296,
|
|
"rewards/rejected": -0.45181978665865385,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.363561630249023,
|
|
"kl": 9.333621978759766,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34649361.47181009,
|
|
"logits/rejected": -36363936.52805281,
|
|
"logps/chosen": -468.34643916913944,
|
|
"logps/rejected": -372.58281765676566,
|
|
"loss": 0.5619,
|
|
"loss/base_kto": 3.2002129554748535,
|
|
"metrics/advantage_var": 480.2209167480469,
|
|
"regularization/lipschitz_norm": 1338.0262451171875,
|
|
"regularization/w1": 1.2952094078063965,
|
|
"rewards/chosen": 0.42227883720963927,
|
|
"rewards/margins": 0.8857319223252798,
|
|
"rewards/rejected": -0.4634530851156405,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.24790382385254,
|
|
"kl": 17.03498649597168,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34141521.245508984,
|
|
"logits/rejected": -35395389.90849673,
|
|
"logps/chosen": -469.58495508982037,
|
|
"logps/rejected": -356.304993872549,
|
|
"loss": 0.5517,
|
|
"loss/base_kto": 3.1420974731445312,
|
|
"metrics/advantage_var": 436.03887939453125,
|
|
"regularization/lipschitz_norm": 1313.9178466796875,
|
|
"regularization/w1": 1.2718724012374878,
|
|
"rewards/chosen": 0.5971917706335376,
|
|
"rewards/margins": 0.8868393404769046,
|
|
"rewards/rejected": -0.28964756984336704,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.500394821166992,
|
|
"kl": 20.996641159057617,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34774957.70015456,
|
|
"logits/rejected": -34581460.32227488,
|
|
"logps/chosen": -474.52347372488407,
|
|
"logps/rejected": -366.91409952606637,
|
|
"loss": 0.5553,
|
|
"loss/base_kto": 3.163972854614258,
|
|
"metrics/advantage_var": 431.9277038574219,
|
|
"regularization/lipschitz_norm": 1320.319091796875,
|
|
"regularization/w1": 1.2780689001083374,
|
|
"rewards/chosen": 0.6257666185429144,
|
|
"rewards/margins": 0.8554037524862521,
|
|
"rewards/rejected": -0.2296371339433378,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 14.845892906188965,
|
|
"kl": 17.076929092407227,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34378566.70476191,
|
|
"logits/rejected": -36923070.62153846,
|
|
"logps/chosen": -470.5708829365079,
|
|
"logps/rejected": -351.86776442307695,
|
|
"loss": 0.563,
|
|
"loss/base_kto": 3.1664352416992188,
|
|
"metrics/advantage_var": 471.05078125,
|
|
"regularization/lipschitz_norm": 1381.47412109375,
|
|
"regularization/w1": 1.3372669219970703,
|
|
"rewards/chosen": 0.6034041147383432,
|
|
"rewards/margins": 0.8736330435244009,
|
|
"rewards/rejected": -0.2702289287860577,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 19.282777786254883,
|
|
"kl": 14.259164810180664,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35830581.68996961,
|
|
"logits/rejected": -36015515.5755627,
|
|
"logps/chosen": -510.91660334346506,
|
|
"logps/rejected": -366.6775020096463,
|
|
"loss": 0.5712,
|
|
"loss/base_kto": 3.2258782386779785,
|
|
"metrics/advantage_var": 456.6734313964844,
|
|
"regularization/lipschitz_norm": 1387.8778076171875,
|
|
"regularization/w1": 1.3434656858444214,
|
|
"rewards/chosen": 0.5217259183843085,
|
|
"rewards/margins": 0.8170433198409881,
|
|
"rewards/rejected": -0.29531740145667956,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 17.025760650634766,
|
|
"kl": 18.5394287109375,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35158551.86766917,
|
|
"logits/rejected": -36200288.15609756,
|
|
"logps/chosen": -478.14699248120303,
|
|
"logps/rejected": -369.25307418699185,
|
|
"loss": 0.5508,
|
|
"loss/base_kto": 3.1588451862335205,
|
|
"metrics/advantage_var": 416.6886901855469,
|
|
"regularization/lipschitz_norm": 1289.041015625,
|
|
"regularization/w1": 1.2477917671203613,
|
|
"rewards/chosen": 0.6125300127760809,
|
|
"rewards/margins": 0.8588316544729178,
|
|
"rewards/rejected": -0.24630164169683688,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 16.39432716369629,
|
|
"kl": 23.536481857299805,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34092204.548031494,
|
|
"logits/rejected": -36183892.53953488,
|
|
"logps/chosen": -484.46929133858265,
|
|
"logps/rejected": -369.04161821705424,
|
|
"loss": 0.5592,
|
|
"loss/base_kto": 3.124847412109375,
|
|
"metrics/advantage_var": 508.1956481933594,
|
|
"regularization/lipschitz_norm": 1393.3729248046875,
|
|
"regularization/w1": 1.3487849235534668,
|
|
"rewards/chosen": 0.7038074403297244,
|
|
"rewards/margins": 0.9230517777812748,
|
|
"rewards/rejected": -0.2192443374515504,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 17.85162353515625,
|
|
"kl": 22.998136520385742,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35762218.666666664,
|
|
"logits/rejected": -36882403.90243903,
|
|
"logps/chosen": -470.07882612179486,
|
|
"logps/rejected": -387.3581364329268,
|
|
"loss": 0.56,
|
|
"loss/base_kto": 3.2242088317871094,
|
|
"metrics/advantage_var": 440.51593017578125,
|
|
"regularization/lipschitz_norm": 1297.2030029296875,
|
|
"regularization/w1": 1.255692481994629,
|
|
"rewards/chosen": 0.6222702417618189,
|
|
"rewards/margins": 0.7873438109898283,
|
|
"rewards/rejected": -0.16507356922800948,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 25.382761001586914,
|
|
"kl": 19.590219497680664,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33445353.8816,
|
|
"logits/rejected": -35821396.03053435,
|
|
"logps/chosen": -490.8473,
|
|
"logps/rejected": -381.35605916030534,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.1736061573028564,
|
|
"metrics/advantage_var": 415.2522888183594,
|
|
"regularization/lipschitz_norm": 1320.162841796875,
|
|
"regularization/w1": 1.2779176235198975,
|
|
"rewards/chosen": 0.583317578125,
|
|
"rewards/margins": 0.838374270932729,
|
|
"rewards/rejected": -0.255056692807729,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 15.28670883178711,
|
|
"kl": 16.164997100830078,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34706287.50920245,
|
|
"logits/rejected": -36549312.40764331,
|
|
"logps/chosen": -481.28513228527606,
|
|
"logps/rejected": -390.67384056528664,
|
|
"loss": 0.5569,
|
|
"loss/base_kto": 3.1864078044891357,
|
|
"metrics/advantage_var": 432.43316650390625,
|
|
"regularization/lipschitz_norm": 1310.327392578125,
|
|
"regularization/w1": 1.2683968544006348,
|
|
"rewards/chosen": 0.5721272661642063,
|
|
"rewards/margins": 0.8397347910603299,
|
|
"rewards/rejected": -0.2676075248961236,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 16.13686752319336,
|
|
"kl": 20.435731887817383,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35811775.61676647,
|
|
"logits/rejected": -36197038.013071895,
|
|
"logps/chosen": -462.16991017964074,
|
|
"logps/rejected": -395.7742800245098,
|
|
"loss": 0.5632,
|
|
"loss/base_kto": 3.184861183166504,
|
|
"metrics/advantage_var": 480.7662048339844,
|
|
"regularization/lipschitz_norm": 1364.7376708984375,
|
|
"regularization/w1": 1.3210660219192505,
|
|
"rewards/chosen": 0.6134483200347353,
|
|
"rewards/margins": 0.8577557944079481,
|
|
"rewards/rejected": -0.24430747437321282,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 24.212142944335938,
|
|
"kl": 20.278656005859375,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35962473.5751938,
|
|
"logits/rejected": -35862179.67874016,
|
|
"logps/chosen": -492.68023255813955,
|
|
"logps/rejected": -361.7607775590551,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.1684212684631348,
|
|
"metrics/advantage_var": 451.9801940917969,
|
|
"regularization/lipschitz_norm": 1348.925537109375,
|
|
"regularization/w1": 1.3057597875595093,
|
|
"rewards/chosen": 0.6777543415394864,
|
|
"rewards/margins": 0.866278002034073,
|
|
"rewards/rejected": -0.1885236604945866,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 17.289684295654297,
|
|
"kl": 18.455638885498047,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35082504.85413534,
|
|
"logits/rejected": -35238290.939837396,
|
|
"logps/chosen": -487.91343984962407,
|
|
"logps/rejected": -394.54552845528457,
|
|
"loss": 0.5683,
|
|
"loss/base_kto": 3.18548846244812,
|
|
"metrics/advantage_var": 485.99786376953125,
|
|
"regularization/lipschitz_norm": 1406.13427734375,
|
|
"regularization/w1": 1.361138105392456,
|
|
"rewards/chosen": 0.623200059474859,
|
|
"rewards/margins": 0.8529667563117492,
|
|
"rewards/rejected": -0.22976669683689024,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 19.184036254882812,
|
|
"kl": 19.55789566040039,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33072462.953271028,
|
|
"logits/rejected": -36717385.028213166,
|
|
"logps/chosen": -503.6235397196262,
|
|
"logps/rejected": -359.8566075626959,
|
|
"loss": 0.557,
|
|
"loss/base_kto": 3.1976475715637207,
|
|
"metrics/advantage_var": 413.19903564453125,
|
|
"regularization/lipschitz_norm": 1299.723876953125,
|
|
"regularization/w1": 1.258132815361023,
|
|
"rewards/chosen": 0.5707444045402551,
|
|
"rewards/margins": 0.8260520236410829,
|
|
"rewards/rejected": -0.2553076191008278,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 13.393630027770996,
|
|
"kl": 14.76126766204834,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34477804.42813455,
|
|
"logits/rejected": -36080558.21086262,
|
|
"logps/chosen": -459.87485665137615,
|
|
"logps/rejected": -376.9496305910543,
|
|
"loss": 0.5588,
|
|
"loss/base_kto": 3.187060594558716,
|
|
"metrics/advantage_var": 448.4735412597656,
|
|
"regularization/lipschitz_norm": 1325.549072265625,
|
|
"regularization/w1": 1.2831313610076904,
|
|
"rewards/chosen": 0.608802818741638,
|
|
"rewards/margins": 0.8601546597023508,
|
|
"rewards/rejected": -0.25135184096071284,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 20.297565460205078,
|
|
"kl": 17.624088287353516,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34961089.70800628,
|
|
"logits/rejected": -36001801.55520996,
|
|
"logps/chosen": -484.5278649921507,
|
|
"logps/rejected": -396.3648668351477,
|
|
"loss": 0.5571,
|
|
"loss/base_kto": 3.154284715652466,
|
|
"metrics/advantage_var": 477.8008728027344,
|
|
"regularization/lipschitz_norm": 1345.2122802734375,
|
|
"regularization/w1": 1.3021655082702637,
|
|
"rewards/chosen": 0.5777575434470663,
|
|
"rewards/margins": 0.9070347125396985,
|
|
"rewards/rejected": -0.3292771690926322,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 14.70542049407959,
|
|
"kl": 15.789746284484863,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35181890.012578614,
|
|
"logits/rejected": -36266766.310559005,
|
|
"logps/chosen": -475.4132272012579,
|
|
"logps/rejected": -403.7750630822981,
|
|
"loss": 0.5656,
|
|
"loss/base_kto": 3.2076053619384766,
|
|
"metrics/advantage_var": 451.7100524902344,
|
|
"regularization/lipschitz_norm": 1360.6304931640625,
|
|
"regularization/w1": 1.317090392112732,
|
|
"rewards/chosen": 0.5488386813949488,
|
|
"rewards/margins": 0.815845928656328,
|
|
"rewards/rejected": -0.2670072472613791,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 18.882680892944336,
|
|
"kl": 13.536798477172852,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -33034973.515055466,
|
|
"logits/rejected": -35850167.42064715,
|
|
"logps/chosen": -510.31784865293184,
|
|
"logps/rejected": -373.64283994607086,
|
|
"loss": 0.5525,
|
|
"loss/base_kto": 3.1619505882263184,
|
|
"metrics/advantage_var": 423.0820007324219,
|
|
"regularization/lipschitz_norm": 1299.3533935546875,
|
|
"regularization/w1": 1.257773995399475,
|
|
"rewards/chosen": 0.551079751572405,
|
|
"rewards/margins": 0.871794834384604,
|
|
"rewards/rejected": -0.32071508281219907,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 17.37849998474121,
|
|
"kl": 13.970664024353027,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34963374.920821115,
|
|
"logits/rejected": -35817728.85618729,
|
|
"logps/chosen": -480.4471224340176,
|
|
"logps/rejected": -381.63827341137124,
|
|
"loss": 0.5578,
|
|
"loss/base_kto": 3.226604461669922,
|
|
"metrics/advantage_var": 408.15692138671875,
|
|
"regularization/lipschitz_norm": 1276.3011474609375,
|
|
"regularization/w1": 1.235459566116333,
|
|
"rewards/chosen": 0.5288357105422928,
|
|
"rewards/margins": 0.8134712224700729,
|
|
"rewards/rejected": -0.2846355119277801,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 19.75193214416504,
|
|
"kl": 14.514517784118652,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35115529.66037736,
|
|
"logits/rejected": -37026784.19875777,
|
|
"logps/chosen": -497.342570754717,
|
|
"logps/rejected": -388.9575892857143,
|
|
"loss": 0.5692,
|
|
"loss/base_kto": 3.1615588665008545,
|
|
"metrics/advantage_var": 509.42437744140625,
|
|
"regularization/lipschitz_norm": 1438.1688232421875,
|
|
"regularization/w1": 1.392147421836853,
|
|
"rewards/chosen": 0.6212985920456221,
|
|
"rewards/margins": 0.8863830660287233,
|
|
"rewards/rejected": -0.2650844739831012,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 19.69506072998047,
|
|
"kl": 16.02850341796875,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34802905.31189711,
|
|
"logits/rejected": -37890903.92705167,
|
|
"logps/chosen": -509.81797628617363,
|
|
"logps/rejected": -335.5232712765957,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.169640302658081,
|
|
"metrics/advantage_var": 452.1182556152344,
|
|
"regularization/lipschitz_norm": 1348.7611083984375,
|
|
"regularization/w1": 1.3056007623672485,
|
|
"rewards/chosen": 0.592926025390625,
|
|
"rewards/margins": 0.8676752246984232,
|
|
"rewards/rejected": -0.27474919930779823,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 16.532896041870117,
|
|
"kl": 16.380760192871094,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34540307.2,
|
|
"logits/rejected": -35429395.2,
|
|
"logps/chosen": -462.204638671875,
|
|
"logps/rejected": -384.1291259765625,
|
|
"loss": 0.5557,
|
|
"loss/base_kto": 3.118773937225342,
|
|
"metrics/advantage_var": 484.7294921875,
|
|
"regularization/lipschitz_norm": 1370.990234375,
|
|
"regularization/w1": 1.3271185159683228,
|
|
"rewards/chosen": 0.6463334560394287,
|
|
"rewards/margins": 0.9337275981903076,
|
|
"rewards/rejected": -0.2873941421508789,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 18.291141510009766,
|
|
"kl": 15.371451377868652,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34378296.529968455,
|
|
"logits/rejected": -36235961.461300306,
|
|
"logps/chosen": -487.8956526025237,
|
|
"logps/rejected": -371.73868034055727,
|
|
"loss": 0.5542,
|
|
"loss/base_kto": 3.1566274166107178,
|
|
"metrics/advantage_var": 443.59869384765625,
|
|
"regularization/lipschitz_norm": 1319.3392333984375,
|
|
"regularization/w1": 1.2771204710006714,
|
|
"rewards/chosen": 0.6160349560084779,
|
|
"rewards/margins": 0.8646803251218922,
|
|
"rewards/rejected": -0.24864536911341428,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 14.354240417480469,
|
|
"kl": 14.69382381439209,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34469841.45454545,
|
|
"logits/rejected": -35720833.54216868,
|
|
"logps/chosen": -470.97326501623377,
|
|
"logps/rejected": -365.2261859939759,
|
|
"loss": 0.5465,
|
|
"loss/base_kto": 3.1806864738464355,
|
|
"metrics/advantage_var": 398.2743835449219,
|
|
"regularization/lipschitz_norm": 1230.29150390625,
|
|
"regularization/w1": 1.1909221410751343,
|
|
"rewards/chosen": 0.5174789924126166,
|
|
"rewards/margins": 0.8392747662060325,
|
|
"rewards/rejected": -0.32179577379341584,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 17.591693878173828,
|
|
"kl": 14.455291748046875,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35108970.146341465,
|
|
"logits/rejected": -36900279.79487179,
|
|
"logps/chosen": -477.1206173780488,
|
|
"logps/rejected": -384.06430288461536,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.2258079051971436,
|
|
"metrics/advantage_var": 416.75469970703125,
|
|
"regularization/lipschitz_norm": 1335.1112060546875,
|
|
"regularization/w1": 1.2923877239227295,
|
|
"rewards/chosen": 0.5574638552782012,
|
|
"rewards/margins": 0.8143197465792829,
|
|
"rewards/rejected": -0.2568558913010817,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 18.717485427856445,
|
|
"kl": 15.758841514587402,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34577203.8585209,
|
|
"logits/rejected": -35298055.00303952,
|
|
"logps/chosen": -501.2543207395498,
|
|
"logps/rejected": -386.2292220744681,
|
|
"loss": 0.5602,
|
|
"loss/base_kto": 3.1767189502716064,
|
|
"metrics/advantage_var": 447.697509765625,
|
|
"regularization/lipschitz_norm": 1348.321533203125,
|
|
"regularization/w1": 1.3051751852035522,
|
|
"rewards/chosen": 0.556097981247488,
|
|
"rewards/margins": 0.8433800031845249,
|
|
"rewards/rejected": -0.28728202193703695,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 15.613844871520996,
|
|
"kl": 14.27246379852295,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34433336.40677966,
|
|
"logits/rejected": -36342087.80982567,
|
|
"logps/chosen": -444.59639830508473,
|
|
"logps/rejected": -366.9770701267829,
|
|
"loss": 0.5546,
|
|
"loss/base_kto": 3.1780214309692383,
|
|
"metrics/advantage_var": 461.02716064453125,
|
|
"regularization/lipschitz_norm": 1300.209228515625,
|
|
"regularization/w1": 1.258602499961853,
|
|
"rewards/chosen": 0.5440258149557011,
|
|
"rewards/margins": 0.8622040915859879,
|
|
"rewards/rejected": -0.31817827663028675,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 24.582273483276367,
|
|
"kl": 13.369653701782227,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35645008.024279214,
|
|
"logits/rejected": -36504762.33172303,
|
|
"logps/chosen": -504.64007966616083,
|
|
"logps/rejected": -373.8888385668277,
|
|
"loss": 0.5646,
|
|
"loss/base_kto": 3.1501927375793457,
|
|
"metrics/advantage_var": 532.1537475585938,
|
|
"regularization/lipschitz_norm": 1411.534912109375,
|
|
"regularization/w1": 1.3663657903671265,
|
|
"rewards/chosen": 0.5976630111022857,
|
|
"rewards/margins": 0.9119348980537753,
|
|
"rewards/rejected": -0.31427188695148955,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 19.878265380859375,
|
|
"kl": 15.078536987304688,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33949032.931419455,
|
|
"logits/rejected": -35318493.89280245,
|
|
"logps/chosen": -476.2298644338118,
|
|
"logps/rejected": -368.3105139739663,
|
|
"loss": 0.5646,
|
|
"loss/base_kto": 3.1518266201019287,
|
|
"metrics/advantage_var": 479.2774353027344,
|
|
"regularization/lipschitz_norm": 1409.8193359375,
|
|
"regularization/w1": 1.3647050857543945,
|
|
"rewards/chosen": 0.5746874649558912,
|
|
"rewards/margins": 0.8874602070476791,
|
|
"rewards/rejected": -0.3127727420917879,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 17.30641746520996,
|
|
"kl": 15.124449729919434,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36250858.80254777,
|
|
"logits/rejected": -36766374.478527606,
|
|
"logps/chosen": -482.51393312101914,
|
|
"logps/rejected": -395.6924127684049,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.1842668056488037,
|
|
"metrics/advantage_var": 449.93511962890625,
|
|
"regularization/lipschitz_norm": 1352.1444091796875,
|
|
"regularization/w1": 1.3088759183883667,
|
|
"rewards/chosen": 0.553188639841262,
|
|
"rewards/margins": 0.8722517445760923,
|
|
"rewards/rejected": -0.31906310473483035,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 22.55718421936035,
|
|
"kl": 14.17529582977295,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33969862.62751159,
|
|
"logits/rejected": -36684643.083728276,
|
|
"logps/chosen": -480.4167310664606,
|
|
"logps/rejected": -387.9507306477093,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.2055633068084717,
|
|
"metrics/advantage_var": 405.29302978515625,
|
|
"regularization/lipschitz_norm": 1295.6785888671875,
|
|
"regularization/w1": 1.2542169094085693,
|
|
"rewards/chosen": 0.5050897848845634,
|
|
"rewards/margins": 0.8096303858947579,
|
|
"rewards/rejected": -0.30454060101019453,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 21.597618103027344,
|
|
"kl": 13.912126541137695,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34215365.3498452,
|
|
"logits/rejected": -36364695.01577287,
|
|
"logps/chosen": -479.19732004643964,
|
|
"logps/rejected": -364.0878844637224,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.1588613986968994,
|
|
"metrics/advantage_var": 448.7057800292969,
|
|
"regularization/lipschitz_norm": 1362.3128662109375,
|
|
"regularization/w1": 1.3187189102172852,
|
|
"rewards/chosen": 0.5791488033330109,
|
|
"rewards/margins": 0.8799048306028255,
|
|
"rewards/rejected": -0.30075602726981465,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 23.34357452392578,
|
|
"kl": 17.03449821472168,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33558875.96116505,
|
|
"logits/rejected": -35319313.01510574,
|
|
"logps/chosen": -500.37570792880257,
|
|
"logps/rejected": -376.7595354984894,
|
|
"loss": 0.5642,
|
|
"loss/base_kto": 3.2193830013275146,
|
|
"metrics/advantage_var": 441.334716796875,
|
|
"regularization/lipschitz_norm": 1336.9698486328125,
|
|
"regularization/w1": 1.2941868305206299,
|
|
"rewards/chosen": 0.5284995662355886,
|
|
"rewards/margins": 0.824237463757008,
|
|
"rewards/rejected": -0.2957378975214195,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_loss": 0.586677935045206,
|
|
"train_runtime": 11045.1759,
|
|
"train_samples_per_second": 13.419,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|