346d69acff
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed2 Source: Original Platform
2229 lines
80 KiB
JSON
2229 lines
80 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 17.28666877746582,
|
|
"kl": 13.869549751281738,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36216319.19370079,
|
|
"logits/rejected": -37507910.25116279,
|
|
"logps/chosen": -481.15551181102364,
|
|
"logps/rejected": -354.4021802325581,
|
|
"loss": 0.6018,
|
|
"loss/base_kto": 3.947984457015991,
|
|
"metrics/advantage_var": 226.25376892089844,
|
|
"regularization/mmd": 0.8666197061538696,
|
|
"regularization/rkhs_norm": 166.97874450683594,
|
|
"rewards/chosen": 0.1351126843550074,
|
|
"rewards/margins": 0.039705591691338554,
|
|
"rewards/rejected": 0.09540709266366884,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 20.494544982910156,
|
|
"kl": 8.738840103149414,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36129686.4,
|
|
"logits/rejected": -36837670.4,
|
|
"logps/chosen": -475.59697265625,
|
|
"logps/rejected": -411.471484375,
|
|
"loss": 0.5951,
|
|
"loss/base_kto": 3.9235122203826904,
|
|
"metrics/advantage_var": 196.90664672851562,
|
|
"regularization/mmd": 0.8370766043663025,
|
|
"regularization/rkhs_norm": 161.28643798828125,
|
|
"rewards/chosen": 0.08794127702713013,
|
|
"rewards/margins": 0.07156163454055786,
|
|
"rewards/rejected": 0.016379642486572265,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 17.606281280517578,
|
|
"kl": 4.159315586090088,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35912865.00628931,
|
|
"logits/rejected": -38222110.21118012,
|
|
"logps/chosen": -488.298054245283,
|
|
"logps/rejected": -373.7237480590062,
|
|
"loss": 0.5808,
|
|
"loss/base_kto": 3.898115634918213,
|
|
"metrics/advantage_var": 159.5330047607422,
|
|
"regularization/mmd": 0.7481209635734558,
|
|
"regularization/rkhs_norm": 144.14663696289062,
|
|
"rewards/chosen": -0.0006746038700799522,
|
|
"rewards/margins": 0.08927492697428267,
|
|
"rewards/rejected": -0.08994953084436262,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 18.236949920654297,
|
|
"kl": 6.960988521575928,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37659216.50314465,
|
|
"logits/rejected": -37593059.37888199,
|
|
"logps/chosen": -504.82360455974845,
|
|
"logps/rejected": -383.9125339673913,
|
|
"loss": 0.5807,
|
|
"loss/base_kto": 3.8643014430999756,
|
|
"metrics/advantage_var": 173.45523071289062,
|
|
"regularization/mmd": 0.7811560034751892,
|
|
"regularization/rkhs_norm": 150.51174926757812,
|
|
"rewards/chosen": 0.08751846409443789,
|
|
"rewards/margins": 0.11913866150904934,
|
|
"rewards/rejected": -0.03162019741461144,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 32.01507568359375,
|
|
"kl": 13.415864944458008,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36480816.64797508,
|
|
"logits/rejected": -37147622.319749214,
|
|
"logps/chosen": -507.1674941588785,
|
|
"logps/rejected": -378.25230211598745,
|
|
"loss": 0.5863,
|
|
"loss/base_kto": 3.8215203285217285,
|
|
"metrics/advantage_var": 216.98648071289062,
|
|
"regularization/mmd": 0.8688768744468689,
|
|
"regularization/rkhs_norm": 167.4136505126953,
|
|
"rewards/chosen": 0.22570339690116337,
|
|
"rewards/margins": 0.1469141373284018,
|
|
"rewards/rejected": 0.07878925957276156,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 18.021196365356445,
|
|
"kl": 9.468263626098633,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35707987.88253478,
|
|
"logits/rejected": -37240802.88151659,
|
|
"logps/chosen": -469.1993817619784,
|
|
"logps/rejected": -370.0141686413902,
|
|
"loss": 0.5801,
|
|
"loss/base_kto": 3.831249237060547,
|
|
"metrics/advantage_var": 192.60733032226562,
|
|
"regularization/mmd": 0.8097471594810486,
|
|
"regularization/rkhs_norm": 156.02061462402344,
|
|
"rewards/chosen": 0.1122589111328125,
|
|
"rewards/margins": 0.1417243198195905,
|
|
"rewards/rejected": -0.02946540868677799,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 18.51321792602539,
|
|
"kl": 7.499949932098389,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36242857.88379205,
|
|
"logits/rejected": -37610872.23003195,
|
|
"logps/chosen": -479.3987958715596,
|
|
"logps/rejected": -384.4207767571885,
|
|
"loss": 0.5826,
|
|
"loss/base_kto": 3.8097259998321533,
|
|
"metrics/advantage_var": 211.20962524414062,
|
|
"regularization/mmd": 0.8511638045310974,
|
|
"regularization/rkhs_norm": 164.00074768066406,
|
|
"rewards/chosen": 0.16908476476640147,
|
|
"rewards/margins": 0.1780336560740007,
|
|
"rewards/rejected": -0.008948891307599247,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 14.697526931762695,
|
|
"kl": 8.385546684265137,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36088274.313846156,
|
|
"logits/rejected": -37605951.39047619,
|
|
"logps/chosen": -493.81947115384617,
|
|
"logps/rejected": -360.7446428571429,
|
|
"loss": 0.5799,
|
|
"loss/base_kto": 3.7885797023773193,
|
|
"metrics/advantage_var": 223.79861450195312,
|
|
"regularization/mmd": 0.8502904176712036,
|
|
"regularization/rkhs_norm": 163.8324432373047,
|
|
"rewards/chosen": 0.15470606877253607,
|
|
"rewards/margins": 0.19735971872332042,
|
|
"rewards/rejected": -0.04265364995078435,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 15.981952667236328,
|
|
"kl": 7.477453708648682,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35705480.586583465,
|
|
"logits/rejected": -36716213.88419405,
|
|
"logps/chosen": -526.872513650546,
|
|
"logps/rejected": -367.46322378716746,
|
|
"loss": 0.5841,
|
|
"loss/base_kto": 3.7968509197235107,
|
|
"metrics/advantage_var": 212.843994140625,
|
|
"regularization/mmd": 0.8761966824531555,
|
|
"regularization/rkhs_norm": 168.8240203857422,
|
|
"rewards/chosen": 0.1464927423390881,
|
|
"rewards/margins": 0.1853673282026582,
|
|
"rewards/rejected": -0.038874585863570094,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 14.56324577331543,
|
|
"kl": 9.05606746673584,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -35997238.022047244,
|
|
"logits/rejected": -37437624.16124031,
|
|
"logps/chosen": -461.77288385826773,
|
|
"logps/rejected": -374.12756782945735,
|
|
"loss": 0.5846,
|
|
"loss/base_kto": 3.8013081550598145,
|
|
"metrics/advantage_var": 252.80130004882812,
|
|
"regularization/mmd": 0.875170886516571,
|
|
"regularization/rkhs_norm": 168.6263885498047,
|
|
"rewards/chosen": 0.1368635282741757,
|
|
"rewards/margins": 0.18972407808264832,
|
|
"rewards/rejected": -0.05286054980847263,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 17.85694694519043,
|
|
"kl": 5.026250839233398,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36942050.119873814,
|
|
"logits/rejected": -38242938.05572756,
|
|
"logps/chosen": -477.1010449526814,
|
|
"logps/rejected": -379.43817724458205,
|
|
"loss": 0.5799,
|
|
"loss/base_kto": 3.8143177032470703,
|
|
"metrics/advantage_var": 197.2425994873047,
|
|
"regularization/mmd": 0.8247326016426086,
|
|
"regularization/rkhs_norm": 158.9080352783203,
|
|
"rewards/chosen": 0.08102270782181893,
|
|
"rewards/margins": 0.17888955745538124,
|
|
"rewards/rejected": -0.0978668496335623,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 16.530935287475586,
|
|
"kl": 2.5676889419555664,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -34537841.365539454,
|
|
"logits/rejected": -37187565.353566006,
|
|
"logps/chosen": -483.52556360708536,
|
|
"logps/rejected": -351.68771339150226,
|
|
"loss": 0.5756,
|
|
"loss/base_kto": 3.769634246826172,
|
|
"metrics/advantage_var": 197.66934204101562,
|
|
"regularization/mmd": 0.8351051211357117,
|
|
"regularization/rkhs_norm": 160.90658569335938,
|
|
"rewards/chosen": -0.024460631292223353,
|
|
"rewards/margins": 0.21369419535681877,
|
|
"rewards/rejected": -0.2381548266490421,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.145092010498047,
|
|
"kl": 5.858470439910889,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37138496.60567823,
|
|
"logits/rejected": -37570468.0619195,
|
|
"logps/chosen": -485.2397969242902,
|
|
"logps/rejected": -390.5070626934984,
|
|
"loss": 0.5842,
|
|
"loss/base_kto": 3.8405380249023438,
|
|
"metrics/advantage_var": 198.77114868164062,
|
|
"regularization/mmd": 0.8329442143440247,
|
|
"regularization/rkhs_norm": 160.49021911621094,
|
|
"rewards/chosen": -0.03661429769233198,
|
|
"rewards/margins": 0.14798289484165117,
|
|
"rewards/rejected": -0.18459719253398316,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 13.552000999450684,
|
|
"kl": 6.439029693603516,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37463869.847094804,
|
|
"logits/rejected": -38733771.65495208,
|
|
"logps/chosen": -493.7569285168196,
|
|
"logps/rejected": -376.71270966453676,
|
|
"loss": 0.5815,
|
|
"loss/base_kto": 3.77490234375,
|
|
"metrics/advantage_var": 218.3898468017578,
|
|
"regularization/mmd": 0.8774310946464539,
|
|
"regularization/rkhs_norm": 169.06187438964844,
|
|
"rewards/chosen": 0.14522417167640242,
|
|
"rewards/margins": 0.22872841888477527,
|
|
"rewards/rejected": -0.08350424720837285,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 16.38521957397461,
|
|
"kl": 9.403640747070312,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -34959307.5712,
|
|
"logits/rejected": -36243984.41526718,
|
|
"logps/chosen": -496.1666,
|
|
"logps/rejected": -396.7485448473282,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.8247368335723877,
|
|
"metrics/advantage_var": 240.5669403076172,
|
|
"regularization/mmd": 0.8972446322441101,
|
|
"regularization/rkhs_norm": 172.8795166015625,
|
|
"rewards/chosen": 0.12554970703125,
|
|
"rewards/margins": 0.13659369605260951,
|
|
"rewards/rejected": -0.011043989021359508,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 15.341755867004395,
|
|
"kl": 8.521685600280762,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35781301.36677116,
|
|
"logits/rejected": -36972237.75700934,
|
|
"logps/chosen": -471.6827978056426,
|
|
"logps/rejected": -365.81563960280374,
|
|
"loss": 0.5803,
|
|
"loss/base_kto": 3.7325761318206787,
|
|
"metrics/advantage_var": 240.1177520751953,
|
|
"regularization/mmd": 0.9098437428474426,
|
|
"regularization/rkhs_norm": 175.3070831298828,
|
|
"rewards/chosen": 0.09740401958597117,
|
|
"rewards/margins": 0.2508578108388917,
|
|
"rewards/rejected": -0.15345379125292055,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.72157096862793,
|
|
"kl": 10.334311485290527,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36954460.98654708,
|
|
"logits/rejected": -38009522.48772504,
|
|
"logps/chosen": -456.5036434977578,
|
|
"logps/rejected": -359.93443126022913,
|
|
"loss": 0.5742,
|
|
"loss/base_kto": 3.677534580230713,
|
|
"metrics/advantage_var": 242.1745147705078,
|
|
"regularization/mmd": 0.9159858822822571,
|
|
"regularization/rkhs_norm": 176.49053955078125,
|
|
"rewards/chosen": 0.1850318965712292,
|
|
"rewards/margins": 0.3090293366908461,
|
|
"rewards/rejected": -0.12399744011961691,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 21.72677993774414,
|
|
"kl": 10.107943534851074,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36391666.87179487,
|
|
"logits/rejected": -36863719.02439024,
|
|
"logps/chosen": -485.4851262019231,
|
|
"logps/rejected": -374.4798971036585,
|
|
"loss": 0.5877,
|
|
"loss/base_kto": 3.8004074096679688,
|
|
"metrics/advantage_var": 234.7443389892578,
|
|
"regularization/mmd": 0.9010725021362305,
|
|
"regularization/rkhs_norm": 173.6170654296875,
|
|
"rewards/chosen": 0.08377578319647373,
|
|
"rewards/margins": 0.15731556360389085,
|
|
"rewards/rejected": -0.07353978040741711,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 13.693880081176758,
|
|
"kl": 9.090913772583008,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -38133170.37672282,
|
|
"logits/rejected": -38013989.56299841,
|
|
"logps/chosen": -482.5569008422665,
|
|
"logps/rejected": -372.1973185805423,
|
|
"loss": 0.5878,
|
|
"loss/base_kto": 3.7692184448242188,
|
|
"metrics/advantage_var": 261.0716247558594,
|
|
"regularization/mmd": 0.9331462979316711,
|
|
"regularization/rkhs_norm": 179.79698181152344,
|
|
"rewards/chosen": 0.18396464606339133,
|
|
"rewards/margins": 0.20749051227831997,
|
|
"rewards/rejected": -0.023525866214928633,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 30.564823150634766,
|
|
"kl": 6.068899154663086,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36919501.136289,
|
|
"logits/rejected": -38799903.284649774,
|
|
"logps/chosen": -492.0698891625616,
|
|
"logps/rejected": -375.616663561848,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.774129629135132,
|
|
"metrics/advantage_var": 229.98155212402344,
|
|
"regularization/mmd": 0.9104791879653931,
|
|
"regularization/rkhs_norm": 175.4295196533203,
|
|
"rewards/chosen": 0.07467767875182804,
|
|
"rewards/margins": 0.20593059861548768,
|
|
"rewards/rejected": -0.13125291986365964,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 15.081184387207031,
|
|
"kl": 7.291744232177734,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36202234.88786482,
|
|
"logits/rejected": -36638667.90461049,
|
|
"logps/chosen": -490.5248655913978,
|
|
"logps/rejected": -367.88473767885534,
|
|
"loss": 0.5782,
|
|
"loss/base_kto": 3.7068536281585693,
|
|
"metrics/advantage_var": 250.29248046875,
|
|
"regularization/mmd": 0.9191361665725708,
|
|
"regularization/rkhs_norm": 177.09751892089844,
|
|
"rewards/chosen": 0.11651709771925403,
|
|
"rewards/margins": 0.2766628800444043,
|
|
"rewards/rejected": -0.16014578232515028,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 16.892141342163086,
|
|
"kl": 8.64151668548584,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -34951623.11111111,
|
|
"logits/rejected": -38028521.48534202,
|
|
"logps/chosen": -507.9345908408408,
|
|
"logps/rejected": -362.2352911237785,
|
|
"loss": 0.5804,
|
|
"loss/base_kto": 3.7336881160736084,
|
|
"metrics/advantage_var": 251.204833984375,
|
|
"regularization/mmd": 0.9091903567314148,
|
|
"regularization/rkhs_norm": 175.18118286132812,
|
|
"rewards/chosen": 0.13266144715271913,
|
|
"rewards/margins": 0.26913717484287847,
|
|
"rewards/rejected": -0.1364757276901593,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 18.60469627380371,
|
|
"kl": 13.147854804992676,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35052245.53945249,
|
|
"logits/rejected": -36744482.57359636,
|
|
"logps/chosen": -472.6548913043478,
|
|
"logps/rejected": -353.2754172989378,
|
|
"loss": 0.581,
|
|
"loss/base_kto": 3.7372944355010986,
|
|
"metrics/advantage_var": 239.1718292236328,
|
|
"regularization/mmd": 0.9105936288833618,
|
|
"regularization/rkhs_norm": 175.4515380859375,
|
|
"rewards/chosen": 0.21827007299844,
|
|
"rewards/margins": 0.23776968821762567,
|
|
"rewards/rejected": -0.019499615219185674,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 14.794768333435059,
|
|
"kl": 6.379868030548096,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36755732.57877813,
|
|
"logits/rejected": -37569875.258358665,
|
|
"logps/chosen": -484.35289389067526,
|
|
"logps/rejected": -388.01676481762917,
|
|
"loss": 0.5774,
|
|
"loss/base_kto": 3.7156808376312256,
|
|
"metrics/advantage_var": 239.91152954101562,
|
|
"regularization/mmd": 0.9038274884223938,
|
|
"regularization/rkhs_norm": 174.1478729248047,
|
|
"rewards/chosen": 0.07609347745153298,
|
|
"rewards/margins": 0.27110358442816673,
|
|
"rewards/rejected": -0.19501010697663373,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.114152908325195,
|
|
"kl": 14.015588760375977,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36049948.53250774,
|
|
"logits/rejected": -37586472.3785489,
|
|
"logps/chosen": -497.7021575077399,
|
|
"logps/rejected": -355.047417192429,
|
|
"loss": 0.5867,
|
|
"loss/base_kto": 3.7359397411346436,
|
|
"metrics/advantage_var": 264.5102844238281,
|
|
"regularization/mmd": 0.9575252532958984,
|
|
"regularization/rkhs_norm": 184.49424743652344,
|
|
"rewards/chosen": 0.16994176339069755,
|
|
"rewards/margins": 0.2273095274515032,
|
|
"rewards/rejected": -0.05736776406080565,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 12.509623527526855,
|
|
"kl": 8.933459281921387,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35814398.422187984,
|
|
"logits/rejected": -36157615.26465927,
|
|
"logps/chosen": -486.26926040061636,
|
|
"logps/rejected": -367.7392531695721,
|
|
"loss": 0.5831,
|
|
"loss/base_kto": 3.7047085762023926,
|
|
"metrics/advantage_var": 290.3509216308594,
|
|
"regularization/mmd": 0.9603023529052734,
|
|
"regularization/rkhs_norm": 185.0293426513672,
|
|
"rewards/chosen": 0.12054084813098878,
|
|
"rewards/margins": 0.2591782034457484,
|
|
"rewards/rejected": -0.13863735531475957,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 18.955705642700195,
|
|
"kl": 16.17554473876953,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37009599.22891566,
|
|
"logits/rejected": -36987328.83116883,
|
|
"logps/chosen": -480.6894766566265,
|
|
"logps/rejected": -357.87243810876623,
|
|
"loss": 0.5762,
|
|
"loss/base_kto": 3.6945130825042725,
|
|
"metrics/advantage_var": 248.5550537109375,
|
|
"regularization/mmd": 0.9150959849357605,
|
|
"regularization/rkhs_norm": 176.31907653808594,
|
|
"rewards/chosen": 0.2735217450612999,
|
|
"rewards/margins": 0.28345770929885983,
|
|
"rewards/rejected": -0.009935964237559925,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 14.894756317138672,
|
|
"kl": 13.262564659118652,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36067758.82926829,
|
|
"logits/rejected": -37874825.84615385,
|
|
"logps/chosen": -467.3940548780488,
|
|
"logps/rejected": -348.3900741185897,
|
|
"loss": 0.5851,
|
|
"loss/base_kto": 3.7631022930145264,
|
|
"metrics/advantage_var": 239.29248046875,
|
|
"regularization/mmd": 0.9173154830932617,
|
|
"regularization/rkhs_norm": 176.74673461914062,
|
|
"rewards/chosen": 0.17195061939518627,
|
|
"rewards/margins": 0.2147787238300555,
|
|
"rewards/rejected": -0.04282810443486923,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 16.39813232421875,
|
|
"kl": 18.698083877563477,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37496603.105882354,
|
|
"logits/rejected": -39177465.17333333,
|
|
"logps/chosen": -473.9681985294118,
|
|
"logps/rejected": -385.46098958333334,
|
|
"loss": 0.5892,
|
|
"loss/base_kto": 3.7126572132110596,
|
|
"metrics/advantage_var": 292.2877502441406,
|
|
"regularization/mmd": 1.0008432865142822,
|
|
"regularization/rkhs_norm": 192.84071350097656,
|
|
"rewards/chosen": 0.31195447585161995,
|
|
"rewards/margins": 0.23376629174924365,
|
|
"rewards/rejected": 0.0781881841023763,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 15.629815101623535,
|
|
"kl": 17.760190963745117,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37011374.08,
|
|
"logits/rejected": -36616374.04444444,
|
|
"logps/chosen": -449.8775961538461,
|
|
"logps/rejected": -393.846378968254,
|
|
"loss": 0.5801,
|
|
"loss/base_kto": 3.748232364654541,
|
|
"metrics/advantage_var": 219.3304901123047,
|
|
"regularization/mmd": 0.8923266530036926,
|
|
"regularization/rkhs_norm": 171.93191528320312,
|
|
"rewards/chosen": 0.2888657789963942,
|
|
"rewards/margins": 0.2183362590524303,
|
|
"rewards/rejected": 0.07052951994396392,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 17.80891227722168,
|
|
"kl": 19.24934196472168,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35685240.75471698,
|
|
"logits/rejected": -37087193.838509314,
|
|
"logps/chosen": -485.37578616352204,
|
|
"logps/rejected": -387.72481560559004,
|
|
"loss": 0.5818,
|
|
"loss/base_kto": 3.7200429439544678,
|
|
"metrics/advantage_var": 246.68115234375,
|
|
"regularization/mmd": 0.9344601631164551,
|
|
"regularization/rkhs_norm": 180.0501251220703,
|
|
"rewards/chosen": 0.3106190423545597,
|
|
"rewards/margins": 0.23427756733575633,
|
|
"rewards/rejected": 0.07634147501880338,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 13.532238960266113,
|
|
"kl": 17.658248901367188,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35347474.197092086,
|
|
"logits/rejected": -36609608.036308624,
|
|
"logps/chosen": -492.469810177706,
|
|
"logps/rejected": -366.7897834720121,
|
|
"loss": 0.5756,
|
|
"loss/base_kto": 3.7010133266448975,
|
|
"metrics/advantage_var": 232.38818359375,
|
|
"regularization/mmd": 0.9040445685386658,
|
|
"regularization/rkhs_norm": 174.18968200683594,
|
|
"rewards/chosen": 0.21897417355045942,
|
|
"rewards/margins": 0.253385691918625,
|
|
"rewards/rejected": -0.03441151836816554,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 15.496357917785645,
|
|
"kl": 11.838883399963379,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36601564.7706422,
|
|
"logits/rejected": -37832164.19169329,
|
|
"logps/chosen": -495.41910359327215,
|
|
"logps/rejected": -371.149660543131,
|
|
"loss": 0.5739,
|
|
"loss/base_kto": 3.676666021347046,
|
|
"metrics/advantage_var": 237.0503387451172,
|
|
"regularization/mmd": 0.91443932056427,
|
|
"regularization/rkhs_norm": 176.19253540039062,
|
|
"rewards/chosen": 0.2042687558981986,
|
|
"rewards/margins": 0.3039905486859881,
|
|
"rewards/rejected": -0.09972179278778953,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 20.567485809326172,
|
|
"kl": 16.626239776611328,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -34921041.1008,
|
|
"logits/rejected": -37017825.123664126,
|
|
"logps/chosen": -473.7086,
|
|
"logps/rejected": -375.5192032442748,
|
|
"loss": 0.5753,
|
|
"loss/base_kto": 3.697774887084961,
|
|
"metrics/advantage_var": 232.10256958007812,
|
|
"regularization/mmd": 0.9045900702476501,
|
|
"regularization/rkhs_norm": 174.2947998046875,
|
|
"rewards/chosen": 0.1565810546875,
|
|
"rewards/margins": 0.2124826243160335,
|
|
"rewards/rejected": -0.055901569628533515,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 12.992965698242188,
|
|
"kl": 9.039730072021484,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -34760422.645367414,
|
|
"logits/rejected": -37357768.41590214,
|
|
"logps/chosen": -484.9376497603834,
|
|
"logps/rejected": -363.50365538990826,
|
|
"loss": 0.5773,
|
|
"loss/base_kto": 3.666606903076172,
|
|
"metrics/advantage_var": 261.6567077636719,
|
|
"regularization/mmd": 0.9521806836128235,
|
|
"regularization/rkhs_norm": 183.46450805664062,
|
|
"rewards/chosen": 0.11359065485457642,
|
|
"rewards/margins": 0.3118521608214629,
|
|
"rewards/rejected": -0.19826150596688646,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 17.397127151489258,
|
|
"kl": 4.5755157470703125,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36470936.5451664,
|
|
"logits/rejected": -37373705.861325115,
|
|
"logps/chosen": -484.4300713153724,
|
|
"logps/rejected": -371.74352369029276,
|
|
"loss": 0.5889,
|
|
"loss/base_kto": 3.7251832485198975,
|
|
"metrics/advantage_var": 285.101806640625,
|
|
"regularization/mmd": 0.9862076640129089,
|
|
"regularization/rkhs_norm": 190.020751953125,
|
|
"rewards/chosen": 0.010507079576350241,
|
|
"rewards/margins": 0.2533064828801108,
|
|
"rewards/rejected": -0.2427994033037606,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.375116348266602,
|
|
"kl": 14.448086738586426,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -37374692.628930815,
|
|
"logits/rejected": -38540908.124223605,
|
|
"logps/chosen": -514.5865762578617,
|
|
"logps/rejected": -353.1389266304348,
|
|
"loss": 0.5739,
|
|
"loss/base_kto": 3.628591299057007,
|
|
"metrics/advantage_var": 268.5576477050781,
|
|
"regularization/mmd": 0.9626501202583313,
|
|
"regularization/rkhs_norm": 185.48171997070312,
|
|
"rewards/chosen": 0.27646538596483144,
|
|
"rewards/margins": 0.3209132078248191,
|
|
"rewards/rejected": -0.04444782185998763,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 16.5850772857666,
|
|
"kl": 8.80722713470459,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -37816466.985645935,
|
|
"logits/rejected": -37114213.53751914,
|
|
"logps/chosen": -489.84370015948963,
|
|
"logps/rejected": -397.30455589586524,
|
|
"loss": 0.5806,
|
|
"loss/base_kto": 3.75372314453125,
|
|
"metrics/advantage_var": 228.4040985107422,
|
|
"regularization/mmd": 0.8908863067626953,
|
|
"regularization/rkhs_norm": 171.6544189453125,
|
|
"rewards/chosen": 0.06759945323402612,
|
|
"rewards/margins": 0.19316826957821281,
|
|
"rewards/rejected": -0.12556881634418668,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 13.62050724029541,
|
|
"kl": 11.619669914245605,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -37364197.218461536,
|
|
"logits/rejected": -37631634.75159235,
|
|
"logps/chosen": -479.99701923076924,
|
|
"logps/rejected": -398.9910429936306,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.5261247158050537,
|
|
"metrics/advantage_var": 315.3587951660156,
|
|
"regularization/mmd": 1.022654414176941,
|
|
"regularization/rkhs_norm": 197.04324340820312,
|
|
"rewards/chosen": 0.25338146503155046,
|
|
"rewards/margins": 0.45228655944555074,
|
|
"rewards/rejected": -0.1989050944140003,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 12.98912239074707,
|
|
"kl": 14.082290649414062,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -37224238.761609904,
|
|
"logits/rejected": -36818217.18611988,
|
|
"logps/chosen": -465.79005417956654,
|
|
"logps/rejected": -384.6505323343849,
|
|
"loss": 0.5685,
|
|
"loss/base_kto": 3.2370293140411377,
|
|
"metrics/advantage_var": 594.7805786132812,
|
|
"regularization/mmd": 1.3111907243728638,
|
|
"regularization/rkhs_norm": 252.6378936767578,
|
|
"rewards/chosen": 0.5340818044940016,
|
|
"rewards/margins": 0.8299278470668895,
|
|
"rewards/rejected": -0.29584604257288794,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 14.384129524230957,
|
|
"kl": 10.371628761291504,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35496298.596721314,
|
|
"logits/rejected": -37267306.22089552,
|
|
"logps/chosen": -490.1142418032787,
|
|
"logps/rejected": -370.1044776119403,
|
|
"loss": 0.5711,
|
|
"loss/base_kto": 3.2188737392425537,
|
|
"metrics/advantage_var": 599.1707763671875,
|
|
"regularization/mmd": 1.350122332572937,
|
|
"regularization/rkhs_norm": 260.13916015625,
|
|
"rewards/chosen": 0.4529114770107582,
|
|
"rewards/margins": 0.8901385952039718,
|
|
"rewards/rejected": -0.43722711819321364,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 13.116654396057129,
|
|
"kl": 7.182905673980713,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36326894.85167464,
|
|
"logits/rejected": -37312913.44563553,
|
|
"logps/chosen": -470.3089114832536,
|
|
"logps/rejected": -356.1921659647779,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.2808244228363037,
|
|
"metrics/advantage_var": 541.5362548828125,
|
|
"regularization/mmd": 1.268153429031372,
|
|
"regularization/rkhs_norm": 244.34556579589844,
|
|
"rewards/chosen": 0.31684688099643643,
|
|
"rewards/margins": 0.7840647396775238,
|
|
"rewards/rejected": -0.4672178586810873,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 11.740735054016113,
|
|
"kl": 8.199923515319824,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34742351.27741936,
|
|
"logits/rejected": -35920880.484848484,
|
|
"logps/chosen": -494.820564516129,
|
|
"logps/rejected": -370.13219696969696,
|
|
"loss": 0.5634,
|
|
"loss/base_kto": 3.1991069316864014,
|
|
"metrics/advantage_var": 542.0139770507812,
|
|
"regularization/mmd": 1.307726263999939,
|
|
"regularization/rkhs_norm": 251.9704132080078,
|
|
"rewards/chosen": 0.3672663042622228,
|
|
"rewards/margins": 0.8765179855033449,
|
|
"rewards/rejected": -0.5092516812411222,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 13.875117301940918,
|
|
"kl": 11.704268455505371,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35606088.700906344,
|
|
"logits/rejected": -36732961.13915858,
|
|
"logps/chosen": -491.577416918429,
|
|
"logps/rejected": -385.93433960355986,
|
|
"loss": 0.5744,
|
|
"loss/base_kto": 3.2271511554718018,
|
|
"metrics/advantage_var": 604.9378051757812,
|
|
"regularization/mmd": 1.368111252784729,
|
|
"regularization/rkhs_norm": 263.605224609375,
|
|
"rewards/chosen": 0.5205691703496743,
|
|
"rewards/margins": 0.858607642645639,
|
|
"rewards/rejected": -0.3380384722959648,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 13.730972290039062,
|
|
"kl": 15.659111022949219,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35645711.802469134,
|
|
"logits/rejected": -36658872.70886076,
|
|
"logps/chosen": -471.9669174382716,
|
|
"logps/rejected": -395.3960393591772,
|
|
"loss": 0.5674,
|
|
"loss/base_kto": 3.1399471759796143,
|
|
"metrics/advantage_var": 651.7294921875,
|
|
"regularization/mmd": 1.3993967771530151,
|
|
"regularization/rkhs_norm": 269.63330078125,
|
|
"rewards/chosen": 0.5837679262514468,
|
|
"rewards/margins": 0.9759252032258955,
|
|
"rewards/rejected": -0.3921572769744487,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 13.653825759887695,
|
|
"kl": 16.984235763549805,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35353943.473354235,
|
|
"logits/rejected": -36772886.33021807,
|
|
"logps/chosen": -472.08801920062695,
|
|
"logps/rejected": -407.0686818535826,
|
|
"loss": 0.5731,
|
|
"loss/base_kto": 3.203996419906616,
|
|
"metrics/advantage_var": 627.2606201171875,
|
|
"regularization/mmd": 1.380798578262329,
|
|
"regularization/rkhs_norm": 266.0498352050781,
|
|
"rewards/chosen": 0.5989840038144103,
|
|
"rewards/margins": 0.8975086880668076,
|
|
"rewards/rejected": -0.2985246842523973,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 14.916421890258789,
|
|
"kl": 18.980070114135742,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35682870.4,
|
|
"logits/rejected": -37265344.0,
|
|
"logps/chosen": -469.505517578125,
|
|
"logps/rejected": -385.406591796875,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.1923668384552,
|
|
"metrics/advantage_var": 793.8643188476562,
|
|
"regularization/mmd": 1.4783775806427002,
|
|
"regularization/rkhs_norm": 284.8511657714844,
|
|
"rewards/chosen": 0.6252220153808594,
|
|
"rewards/margins": 0.9321086883544922,
|
|
"rewards/rejected": -0.3068866729736328,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 13.798221588134766,
|
|
"kl": 8.175427436828613,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35857702.835913315,
|
|
"logits/rejected": -36561348.239747636,
|
|
"logps/chosen": -480.4967105263158,
|
|
"logps/rejected": -369.85996648264984,
|
|
"loss": 0.5657,
|
|
"loss/base_kto": 3.243157148361206,
|
|
"metrics/advantage_var": 499.720703125,
|
|
"regularization/mmd": 1.2824822664260864,
|
|
"regularization/rkhs_norm": 247.10643005371094,
|
|
"rewards/chosen": 0.3872545555280089,
|
|
"rewards/margins": 0.8420906578421847,
|
|
"rewards/rejected": -0.45483610231417587,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 9.352314949035645,
|
|
"kl": 23.32759666442871,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -37269526.47021943,
|
|
"logits/rejected": -36725230.45482866,
|
|
"logps/chosen": -470.67584247648904,
|
|
"logps/rejected": -372.3608352803738,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.1387908458709717,
|
|
"metrics/advantage_var": 631.537841796875,
|
|
"regularization/mmd": 1.399050235748291,
|
|
"regularization/rkhs_norm": 269.5664978027344,
|
|
"rewards/chosen": 0.7032140175750636,
|
|
"rewards/margins": 0.9272310330738103,
|
|
"rewards/rejected": -0.22401701549874659,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 12.985386848449707,
|
|
"kl": 12.752604484558105,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35332991.60856269,
|
|
"logits/rejected": -35242435.47603834,
|
|
"logps/chosen": -481.59121750764524,
|
|
"logps/rejected": -364.62460063897765,
|
|
"loss": 0.56,
|
|
"loss/base_kto": 3.198040008544922,
|
|
"metrics/advantage_var": 520.0338134765625,
|
|
"regularization/mmd": 1.2815788984298706,
|
|
"regularization/rkhs_norm": 246.9323272705078,
|
|
"rewards/chosen": 0.5117893510637662,
|
|
"rewards/margins": 0.8754246244700661,
|
|
"rewards/rejected": -0.3636352734062999,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 14.315081596374512,
|
|
"kl": 15.352499961853027,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -36456540.78964401,
|
|
"logits/rejected": -38448242.465256795,
|
|
"logps/chosen": -486.46677791262135,
|
|
"logps/rejected": -364.0645770392749,
|
|
"loss": 0.5649,
|
|
"loss/base_kto": 3.1950738430023193,
|
|
"metrics/advantage_var": 526.9627075195312,
|
|
"regularization/mmd": 1.3240159749984741,
|
|
"regularization/rkhs_norm": 255.1090850830078,
|
|
"rewards/chosen": 0.5178439933505259,
|
|
"rewards/margins": 0.8641094807909291,
|
|
"rewards/rejected": -0.34626548744040314,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 11.546377182006836,
|
|
"kl": 13.232007026672363,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -36010176.60188088,
|
|
"logits/rejected": -37478313.86915888,
|
|
"logps/chosen": -479.94494514106583,
|
|
"logps/rejected": -396.6027550623053,
|
|
"loss": 0.5692,
|
|
"loss/base_kto": 3.1710774898529053,
|
|
"metrics/advantage_var": 709.99072265625,
|
|
"regularization/mmd": 1.382637858390808,
|
|
"regularization/rkhs_norm": 266.4042053222656,
|
|
"rewards/chosen": 0.5016709091521355,
|
|
"rewards/margins": 0.8928089562248818,
|
|
"rewards/rejected": -0.3911380470727463,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 10.159113883972168,
|
|
"kl": 12.148736953735352,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -36512554.73449921,
|
|
"logits/rejected": -38063398.14439324,
|
|
"logps/chosen": -474.3939785373609,
|
|
"logps/rejected": -374.65101766513055,
|
|
"loss": 0.5712,
|
|
"loss/base_kto": 3.201780080795288,
|
|
"metrics/advantage_var": 608.3012084960938,
|
|
"regularization/mmd": 1.3678390979766846,
|
|
"regularization/rkhs_norm": 263.5528259277344,
|
|
"rewards/chosen": 0.5240032153592011,
|
|
"rewards/margins": 0.8929486403664015,
|
|
"rewards/rejected": -0.36894542500720046,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 15.198099136352539,
|
|
"kl": 12.118966102600098,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35891424.39506173,
|
|
"logits/rejected": -36746252.962025315,
|
|
"logps/chosen": -492.47202932098764,
|
|
"logps/rejected": -367.29756724683546,
|
|
"loss": 0.5676,
|
|
"loss/base_kto": 3.21653413772583,
|
|
"metrics/advantage_var": 601.02294921875,
|
|
"regularization/mmd": 1.324424147605896,
|
|
"regularization/rkhs_norm": 255.1876983642578,
|
|
"rewards/chosen": 0.48597100340289834,
|
|
"rewards/margins": 0.839893086810916,
|
|
"rewards/rejected": -0.35392208340801773,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 23.502729415893555,
|
|
"kl": 12.213374137878418,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35316529.34939759,
|
|
"logits/rejected": -36814023.48051948,
|
|
"logps/chosen": -476.14664909638554,
|
|
"logps/rejected": -362.5178064123377,
|
|
"loss": 0.566,
|
|
"loss/base_kto": 3.2537143230438232,
|
|
"metrics/advantage_var": 508.3929748535156,
|
|
"regularization/mmd": 1.2744741439819336,
|
|
"regularization/rkhs_norm": 245.5634307861328,
|
|
"rewards/chosen": 0.5508395275437689,
|
|
"rewards/margins": 0.8143075887184341,
|
|
"rewards/rejected": -0.2634680611746652,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 13.538008689880371,
|
|
"kl": 15.613471031188965,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34546725.236363634,
|
|
"logits/rejected": -36143731.61290322,
|
|
"logps/chosen": -473.5277935606061,
|
|
"logps/rejected": -386.2426411290323,
|
|
"loss": 0.5634,
|
|
"loss/base_kto": 3.24967885017395,
|
|
"metrics/advantage_var": 471.4722595214844,
|
|
"regularization/mmd": 1.257878303527832,
|
|
"regularization/rkhs_norm": 242.3657684326172,
|
|
"rewards/chosen": 0.5266698663884943,
|
|
"rewards/margins": 0.7974940772629904,
|
|
"rewards/rejected": -0.270824210874496,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 16.916542053222656,
|
|
"kl": 15.56737995147705,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34568906.49517685,
|
|
"logits/rejected": -35735589.34954407,
|
|
"logps/chosen": -492.60651125401927,
|
|
"logps/rejected": -358.8301671732523,
|
|
"loss": 0.5585,
|
|
"loss/base_kto": 3.189795732498169,
|
|
"metrics/advantage_var": 529.7860717773438,
|
|
"regularization/mmd": 1.2780075073242188,
|
|
"regularization/rkhs_norm": 246.2442169189453,
|
|
"rewards/chosen": 0.5155350173017986,
|
|
"rewards/margins": 0.8675446601144173,
|
|
"rewards/rejected": -0.3520096428126187,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 12.518861770629883,
|
|
"kl": 7.532243251800537,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34510080.40829346,
|
|
"logits/rejected": -36870633.2618683,
|
|
"logps/chosen": -492.5997807017544,
|
|
"logps/rejected": -385.3143903139357,
|
|
"loss": 0.5723,
|
|
"loss/base_kto": 3.241284132003784,
|
|
"metrics/advantage_var": 557.99072265625,
|
|
"regularization/mmd": 1.3371669054031372,
|
|
"regularization/rkhs_norm": 257.6429443359375,
|
|
"rewards/chosen": 0.408393920703748,
|
|
"rewards/margins": 0.8317354468929709,
|
|
"rewards/rejected": -0.4233415261892228,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 13.507658004760742,
|
|
"kl": 11.165776252746582,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34689529.7939394,
|
|
"logits/rejected": -37316611.30322581,
|
|
"logps/chosen": -496.69389204545456,
|
|
"logps/rejected": -379.5487147177419,
|
|
"loss": 0.5758,
|
|
"loss/base_kto": 3.2763335704803467,
|
|
"metrics/advantage_var": 578.8201293945312,
|
|
"regularization/mmd": 1.330436110496521,
|
|
"regularization/rkhs_norm": 256.3460388183594,
|
|
"rewards/chosen": 0.46783918900923294,
|
|
"rewards/margins": 0.8089585245529578,
|
|
"rewards/rejected": -0.3411193355437248,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 12.720839500427246,
|
|
"kl": 14.162638664245605,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34116387.395498395,
|
|
"logits/rejected": -37287929.77507599,
|
|
"logps/chosen": -483.574959807074,
|
|
"logps/rejected": -360.15197568389056,
|
|
"loss": 0.5673,
|
|
"loss/base_kto": 3.2285149097442627,
|
|
"metrics/advantage_var": 568.7373657226562,
|
|
"regularization/mmd": 1.309622049331665,
|
|
"regularization/rkhs_norm": 252.3356475830078,
|
|
"rewards/chosen": 0.5008895726832546,
|
|
"rewards/margins": 0.843100522056355,
|
|
"rewards/rejected": -0.3422109493731003,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 10.554686546325684,
|
|
"kl": 15.69724178314209,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34917303.66091052,
|
|
"logits/rejected": -35976059.81959564,
|
|
"logps/chosen": -457.6415816326531,
|
|
"logps/rejected": -373.05538005443236,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.2105298042297363,
|
|
"metrics/advantage_var": 482.99664306640625,
|
|
"regularization/mmd": 1.2578706741333008,
|
|
"regularization/rkhs_norm": 242.36428833007812,
|
|
"rewards/chosen": 0.5146402930913953,
|
|
"rewards/margins": 0.851425189823328,
|
|
"rewards/rejected": -0.33678489673193285,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 12.624411582946777,
|
|
"kl": 18.55557632446289,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35022171.62519201,
|
|
"logits/rejected": -35563142.308426075,
|
|
"logps/chosen": -483.1249039938556,
|
|
"logps/rejected": -375.14288553259144,
|
|
"loss": 0.5618,
|
|
"loss/base_kto": 3.19071626663208,
|
|
"metrics/advantage_var": 546.0989379882812,
|
|
"regularization/mmd": 1.3034061193466187,
|
|
"regularization/rkhs_norm": 251.1379852294922,
|
|
"rewards/chosen": 0.6021119570402506,
|
|
"rewards/margins": 0.8761229352174167,
|
|
"rewards/rejected": -0.27401097817716613,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.09439468383789,
|
|
"kl": 11.00549602508545,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -36292341.85736925,
|
|
"logits/rejected": -36865038.20030817,
|
|
"logps/chosen": -484.70701267828844,
|
|
"logps/rejected": -381.20040446841296,
|
|
"loss": 0.5723,
|
|
"loss/base_kto": 3.2627198696136475,
|
|
"metrics/advantage_var": 536.9502563476562,
|
|
"regularization/mmd": 1.3156884908676147,
|
|
"regularization/rkhs_norm": 253.50454711914062,
|
|
"rewards/chosen": 0.43750696439183834,
|
|
"rewards/margins": 0.7933569254948574,
|
|
"rewards/rejected": -0.3558499611030191,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 15.669052124023438,
|
|
"kl": 14.620813369750977,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35399084.8,
|
|
"logits/rejected": -36422662.4,
|
|
"logps/chosen": -490.25322265625,
|
|
"logps/rejected": -379.8054931640625,
|
|
"loss": 0.5619,
|
|
"loss/base_kto": 3.240927219390869,
|
|
"metrics/advantage_var": 489.51904296875,
|
|
"regularization/mmd": 1.2539480924606323,
|
|
"regularization/rkhs_norm": 241.6085205078125,
|
|
"rewards/chosen": 0.4669234275817871,
|
|
"rewards/margins": 0.7803276538848878,
|
|
"rewards/rejected": -0.3134042263031006,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 13.812790870666504,
|
|
"kl": 16.63750648498535,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -36057573.86186186,
|
|
"logits/rejected": -36731967.37459283,
|
|
"logps/chosen": -467.51318506006004,
|
|
"logps/rejected": -372.22892915309444,
|
|
"loss": 0.5633,
|
|
"loss/base_kto": 3.2285385131835938,
|
|
"metrics/advantage_var": 585.1842041015625,
|
|
"regularization/mmd": 1.277657151222229,
|
|
"regularization/rkhs_norm": 246.1767120361328,
|
|
"rewards/chosen": 0.6083782757366741,
|
|
"rewards/margins": 0.8179173599071368,
|
|
"rewards/rejected": -0.20953908417046263,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 12.299062728881836,
|
|
"kl": 12.596667289733887,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -36139868.28840125,
|
|
"logits/rejected": -38455142.87850467,
|
|
"logps/chosen": -481.437059169279,
|
|
"logps/rejected": -380.82138337227417,
|
|
"loss": 0.5702,
|
|
"loss/base_kto": 3.2609164714813232,
|
|
"metrics/advantage_var": 542.4650268554688,
|
|
"regularization/mmd": 1.301024317741394,
|
|
"regularization/rkhs_norm": 250.6790313720703,
|
|
"rewards/chosen": 0.5080097162611432,
|
|
"rewards/margins": 0.7714715333518997,
|
|
"rewards/rejected": -0.26346181709075644,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 14.116256713867188,
|
|
"kl": 15.492639541625977,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34781736.23420647,
|
|
"logits/rejected": -37028602.725832015,
|
|
"logps/chosen": -481.3108628659476,
|
|
"logps/rejected": -358.75680962757525,
|
|
"loss": 0.5497,
|
|
"loss/base_kto": 3.203596830368042,
|
|
"metrics/advantage_var": 437.26300048828125,
|
|
"regularization/mmd": 1.193867564201355,
|
|
"regularization/rkhs_norm": 230.0322723388672,
|
|
"rewards/chosen": 0.5390303366356896,
|
|
"rewards/margins": 0.8165800440151592,
|
|
"rewards/rejected": -0.2775497073794696,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 15.030481338500977,
|
|
"kl": 11.008419036865234,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35722243.210031345,
|
|
"logits/rejected": -35698132.93457944,
|
|
"logps/chosen": -470.43172021943576,
|
|
"logps/rejected": -367.1655957943925,
|
|
"loss": 0.5676,
|
|
"loss/base_kto": 3.234706163406372,
|
|
"metrics/advantage_var": 517.1394653320312,
|
|
"regularization/mmd": 1.306067705154419,
|
|
"regularization/rkhs_norm": 251.6508026123047,
|
|
"rewards/chosen": 0.47645769747073374,
|
|
"rewards/margins": 0.8401498652341318,
|
|
"rewards/rejected": -0.3636921677633981,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 13.965100288391113,
|
|
"kl": 10.82063102722168,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -36113708.39546191,
|
|
"logits/rejected": -36955693.56259427,
|
|
"logps/chosen": -496.9145563209076,
|
|
"logps/rejected": -382.8325791855204,
|
|
"loss": 0.5697,
|
|
"loss/base_kto": 3.221923589706421,
|
|
"metrics/advantage_var": 579.80224609375,
|
|
"regularization/mmd": 1.3358854055404663,
|
|
"regularization/rkhs_norm": 257.39605712890625,
|
|
"rewards/chosen": 0.4650991183239212,
|
|
"rewards/margins": 0.8364370549784508,
|
|
"rewards/rejected": -0.3713379366545296,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 13.135164260864258,
|
|
"kl": 11.971667289733887,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -37069436.937799044,
|
|
"logits/rejected": -36610599.987748854,
|
|
"logps/chosen": -501.1855562200957,
|
|
"logps/rejected": -386.27837863705975,
|
|
"loss": 0.5612,
|
|
"loss/base_kto": 3.246180772781372,
|
|
"metrics/advantage_var": 493.3094787597656,
|
|
"regularization/mmd": 1.2436574697494507,
|
|
"regularization/rkhs_norm": 239.62571716308594,
|
|
"rewards/chosen": 0.45027951570599084,
|
|
"rewards/margins": 0.791760629577964,
|
|
"rewards/rejected": -0.3414811138719731,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 12.561501502990723,
|
|
"kl": 8.76456356048584,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34343353.35193799,
|
|
"logits/rejected": -35630023.55905512,
|
|
"logps/chosen": -477.0470930232558,
|
|
"logps/rejected": -397.0420275590551,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.1628811359405518,
|
|
"metrics/advantage_var": 565.871337890625,
|
|
"regularization/mmd": 1.3064866065979004,
|
|
"regularization/rkhs_norm": 251.73155212402344,
|
|
"rewards/chosen": 0.4099231808684593,
|
|
"rewards/margins": 0.9257905433038678,
|
|
"rewards/rejected": -0.5158673624354084,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 27.34815216064453,
|
|
"kl": 16.869966506958008,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35392244.38184664,
|
|
"logits/rejected": -35469979.756630264,
|
|
"logps/chosen": -476.41035798122067,
|
|
"logps/rejected": -366.750609399376,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.191033124923706,
|
|
"metrics/advantage_var": 543.9557495117188,
|
|
"regularization/mmd": 1.2676764726638794,
|
|
"regularization/rkhs_norm": 244.253662109375,
|
|
"rewards/chosen": 0.5922295655443075,
|
|
"rewards/margins": 0.8774939125195538,
|
|
"rewards/rejected": -0.2852643469752462,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 10.379315376281738,
|
|
"kl": 10.070303916931152,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -36206361.6,
|
|
"logits/rejected": -35660636.8,
|
|
"logps/chosen": -488.068408203125,
|
|
"logps/rejected": -367.74365234375,
|
|
"loss": 0.5692,
|
|
"loss/base_kto": 3.191129684448242,
|
|
"metrics/advantage_var": 589.2523803710938,
|
|
"regularization/mmd": 1.3624193668365479,
|
|
"regularization/rkhs_norm": 262.508544921875,
|
|
"rewards/chosen": 0.4839737892150879,
|
|
"rewards/margins": 0.8736880779266357,
|
|
"rewards/rejected": -0.38971428871154784,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 10.132719039916992,
|
|
"kl": 8.817431449890137,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35998789.96273292,
|
|
"logits/rejected": -35578049.20754717,
|
|
"logps/chosen": -475.36733307453414,
|
|
"logps/rejected": -383.6253439465409,
|
|
"loss": 0.5637,
|
|
"loss/base_kto": 3.250783920288086,
|
|
"metrics/advantage_var": 499.1070251464844,
|
|
"regularization/mmd": 1.258469820022583,
|
|
"regularization/rkhs_norm": 242.479736328125,
|
|
"rewards/chosen": 0.4158513797736316,
|
|
"rewards/margins": 0.8038303713960766,
|
|
"rewards/rejected": -0.387978991622445,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 11.175532341003418,
|
|
"kl": 12.814590454101562,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34154521.20615385,
|
|
"logits/rejected": -35270460.952380955,
|
|
"logps/chosen": -494.93375,
|
|
"logps/rejected": -365.18846726190475,
|
|
"loss": 0.5665,
|
|
"loss/base_kto": 3.188995361328125,
|
|
"metrics/advantage_var": 550.5657958984375,
|
|
"regularization/mmd": 1.343163013458252,
|
|
"regularization/rkhs_norm": 258.79827880859375,
|
|
"rewards/chosen": 0.5832626108022837,
|
|
"rewards/margins": 0.8485274307195084,
|
|
"rewards/rejected": -0.2652648199172247,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 10.874167442321777,
|
|
"kl": 10.524325370788574,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -36449446.84984025,
|
|
"logits/rejected": -37267568.73394495,
|
|
"logps/chosen": -480.68310702875397,
|
|
"logps/rejected": -368.5727255351682,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.2181408405303955,
|
|
"metrics/advantage_var": 512.5886840820312,
|
|
"regularization/mmd": 1.2961418628692627,
|
|
"regularization/rkhs_norm": 249.73828125,
|
|
"rewards/chosen": 0.42515188588882785,
|
|
"rewards/margins": 0.8297866007445236,
|
|
"rewards/rejected": -0.40463471485569574,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 12.937446594238281,
|
|
"kl": 11.89905834197998,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35112049.06998445,
|
|
"logits/rejected": -35724656.1255887,
|
|
"logps/chosen": -496.1260206065319,
|
|
"logps/rejected": -373.39602138932497,
|
|
"loss": 0.5667,
|
|
"loss/base_kto": 3.214163303375244,
|
|
"metrics/advantage_var": 615.8258056640625,
|
|
"regularization/mmd": 1.3195157051086426,
|
|
"regularization/rkhs_norm": 254.241943359375,
|
|
"rewards/chosen": 0.4987821920107164,
|
|
"rewards/margins": 0.8687606850230054,
|
|
"rewards/rejected": -0.369978493012289,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 10.794614791870117,
|
|
"kl": 13.87277889251709,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35004384.540706605,
|
|
"logits/rejected": -36439240.880382776,
|
|
"logps/chosen": -493.66599462365593,
|
|
"logps/rejected": -364.7004336124402,
|
|
"loss": 0.5356,
|
|
"loss/base_kto": 3.0849404335021973,
|
|
"metrics/advantage_var": 414.8458557128906,
|
|
"regularization/mmd": 1.2002458572387695,
|
|
"regularization/rkhs_norm": 231.26126098632812,
|
|
"rewards/chosen": 0.5897390714255712,
|
|
"rewards/margins": 0.9555537427974936,
|
|
"rewards/rejected": -0.36581467137192236,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 12.27173137664795,
|
|
"kl": 11.178915977478027,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34765633.084745765,
|
|
"logits/rejected": -35100467.524564184,
|
|
"logps/chosen": -476.7357473035439,
|
|
"logps/rejected": -405.81658577654514,
|
|
"loss": 0.5304,
|
|
"loss/base_kto": 3.0577104091644287,
|
|
"metrics/advantage_var": 396.5050048828125,
|
|
"regularization/mmd": 1.1852072477340698,
|
|
"regularization/rkhs_norm": 228.3636474609375,
|
|
"rewards/chosen": 0.5489674995786787,
|
|
"rewards/margins": 0.9826593314304022,
|
|
"rewards/rejected": -0.43369183185172344,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 13.18808650970459,
|
|
"kl": 15.312929153442383,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33659943.203675345,
|
|
"logits/rejected": -35243408.12759171,
|
|
"logps/chosen": -491.76722817764164,
|
|
"logps/rejected": -389.0051086523126,
|
|
"loss": 0.5251,
|
|
"loss/base_kto": 3.044113874435425,
|
|
"metrics/advantage_var": 385.6358642578125,
|
|
"regularization/mmd": 1.1567519903182983,
|
|
"regularization/rkhs_norm": 222.88096618652344,
|
|
"rewards/chosen": 0.6191242679520004,
|
|
"rewards/margins": 0.9529156083190757,
|
|
"rewards/rejected": -0.33379134036707536,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 13.835762023925781,
|
|
"kl": 12.04203987121582,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -36170506.720268,
|
|
"logits/rejected": -36780595.724743776,
|
|
"logps/chosen": -481.39175041876047,
|
|
"logps/rejected": -405.1716233528551,
|
|
"loss": 0.5212,
|
|
"loss/base_kto": 3.049400806427002,
|
|
"metrics/advantage_var": 360.25384521484375,
|
|
"regularization/mmd": 1.1205371618270874,
|
|
"regularization/rkhs_norm": 215.9031219482422,
|
|
"rewards/chosen": 0.5269861269236809,
|
|
"rewards/margins": 0.9735436368751815,
|
|
"rewards/rejected": -0.44655750995150073,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 10.22216510772705,
|
|
"kl": 9.879226684570312,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33514864.252365932,
|
|
"logits/rejected": -35528998.835913315,
|
|
"logps/chosen": -479.18661277602524,
|
|
"logps/rejected": -365.45714009287923,
|
|
"loss": 0.5247,
|
|
"loss/base_kto": 3.052150011062622,
|
|
"metrics/advantage_var": 372.58612060546875,
|
|
"regularization/mmd": 1.1455343961715698,
|
|
"regularization/rkhs_norm": 220.7195281982422,
|
|
"rewards/chosen": 0.5359188043732749,
|
|
"rewards/margins": 0.9770049504887934,
|
|
"rewards/rejected": -0.44108614611551855,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 12.031999588012695,
|
|
"kl": 11.795473098754883,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35341678.05984252,
|
|
"logits/rejected": -34914402.43100775,
|
|
"logps/chosen": -479.39207677165354,
|
|
"logps/rejected": -380.02703488372094,
|
|
"loss": 0.5262,
|
|
"loss/base_kto": 3.060995578765869,
|
|
"metrics/advantage_var": 370.60595703125,
|
|
"regularization/mmd": 1.1483268737792969,
|
|
"regularization/rkhs_norm": 221.25759887695312,
|
|
"rewards/chosen": 0.5401754544475886,
|
|
"rewards/margins": 0.9658052047806798,
|
|
"rewards/rejected": -0.4256297503330911,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 7.272770881652832,
|
|
"kl": 12.063289642333984,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35471808.28785607,
|
|
"logits/rejected": -37075782.57748777,
|
|
"logps/chosen": -483.42789542728633,
|
|
"logps/rejected": -373.52492862969007,
|
|
"loss": 0.5359,
|
|
"loss/base_kto": 3.1197516918182373,
|
|
"metrics/advantage_var": 397.1454772949219,
|
|
"regularization/mmd": 1.1671723127365112,
|
|
"regularization/rkhs_norm": 224.88870239257812,
|
|
"rewards/chosen": 0.5531095536216267,
|
|
"rewards/margins": 0.9306453005087342,
|
|
"rewards/rejected": -0.37753574688710745,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 10.803357124328613,
|
|
"kl": 18.992996215820312,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35726013.05863708,
|
|
"logits/rejected": -35897009.50385208,
|
|
"logps/chosen": -477.5151545166403,
|
|
"logps/rejected": -355.7731847072419,
|
|
"loss": 0.5222,
|
|
"loss/base_kto": 3.07306170463562,
|
|
"metrics/advantage_var": 355.2674560546875,
|
|
"regularization/mmd": 1.1047872304916382,
|
|
"regularization/rkhs_norm": 212.8684539794922,
|
|
"rewards/chosen": 0.6384236234492621,
|
|
"rewards/margins": 0.9141885942727646,
|
|
"rewards/rejected": -0.2757649708235025,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 12.077045440673828,
|
|
"kl": 10.724763870239258,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33900746.54945055,
|
|
"logits/rejected": -34756350.009331256,
|
|
"logps/chosen": -454.79341640502355,
|
|
"logps/rejected": -357.78628013219287,
|
|
"loss": 0.5241,
|
|
"loss/base_kto": 3.111751079559326,
|
|
"metrics/advantage_var": 340.6296081542969,
|
|
"regularization/mmd": 1.0811723470687866,
|
|
"regularization/rkhs_norm": 208.31838989257812,
|
|
"rewards/chosen": 0.4972784260939953,
|
|
"rewards/margins": 0.925645127211195,
|
|
"rewards/rejected": -0.42836670111719966,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 13.94926643371582,
|
|
"kl": 12.412457466125488,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33723913.66037736,
|
|
"logits/rejected": -36846881.39130435,
|
|
"logps/chosen": -471.5267786949685,
|
|
"logps/rejected": -375.14635093167703,
|
|
"loss": 0.5255,
|
|
"loss/base_kto": 3.0692825317382812,
|
|
"metrics/advantage_var": 370.166015625,
|
|
"regularization/mmd": 1.1345977783203125,
|
|
"regularization/rkhs_norm": 218.6123046875,
|
|
"rewards/chosen": 0.563676654167895,
|
|
"rewards/margins": 0.9501858801435477,
|
|
"rewards/rejected": -0.3865092259756527,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 11.850485801696777,
|
|
"kl": 9.79004192352295,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34795158.5882353,
|
|
"logits/rejected": -36215982.59907834,
|
|
"logps/chosen": -492.7114467408585,
|
|
"logps/rejected": -373.8477822580645,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.0690057277679443,
|
|
"metrics/advantage_var": 401.89691162109375,
|
|
"regularization/mmd": 1.1717764139175415,
|
|
"regularization/rkhs_norm": 225.7758026123047,
|
|
"rewards/chosen": 0.5096414199125596,
|
|
"rewards/margins": 0.9560450991230292,
|
|
"rewards/rejected": -0.4464036792104695,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 9.599963188171387,
|
|
"kl": 11.842473030090332,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -36198355.616099074,
|
|
"logits/rejected": -34933575.873817034,
|
|
"logps/chosen": -469.62267801857587,
|
|
"logps/rejected": -392.19395701892745,
|
|
"loss": 0.528,
|
|
"loss/base_kto": 3.074521780014038,
|
|
"metrics/advantage_var": 387.1634216308594,
|
|
"regularization/mmd": 1.1494674682617188,
|
|
"regularization/rkhs_norm": 221.4773406982422,
|
|
"rewards/chosen": 0.550881636770148,
|
|
"rewards/margins": 0.9560462892124785,
|
|
"rewards/rejected": -0.40516465244233046,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 12.557998657226562,
|
|
"kl": 11.34885025024414,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34241445.46341463,
|
|
"logits/rejected": -36624948.51282051,
|
|
"logps/chosen": -456.6895960365854,
|
|
"logps/rejected": -374.07882612179486,
|
|
"loss": 0.5255,
|
|
"loss/base_kto": 3.1066834926605225,
|
|
"metrics/advantage_var": 336.653076171875,
|
|
"regularization/mmd": 1.0975435972213745,
|
|
"regularization/rkhs_norm": 211.47276306152344,
|
|
"rewards/chosen": 0.5479634447795588,
|
|
"rewards/margins": 0.8984530611736018,
|
|
"rewards/rejected": -0.35048961639404297,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 9.489789009094238,
|
|
"kl": 13.431262016296387,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36068473.018181816,
|
|
"logits/rejected": -37891844.95483871,
|
|
"logps/chosen": -501.8767518939394,
|
|
"logps/rejected": -363.4361391129032,
|
|
"loss": 0.5271,
|
|
"loss/base_kto": 3.0921616554260254,
|
|
"metrics/advantage_var": 364.2037048339844,
|
|
"regularization/mmd": 1.1243157386779785,
|
|
"regularization/rkhs_norm": 216.6311492919922,
|
|
"rewards/chosen": 0.5953232273910984,
|
|
"rewards/margins": 0.9308806604886917,
|
|
"rewards/rejected": -0.3355574330975932,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 13.395483016967773,
|
|
"kl": 15.368057250976562,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34993718.80122324,
|
|
"logits/rejected": -36962902.69648562,
|
|
"logps/chosen": -499.00095565749234,
|
|
"logps/rejected": -394.44473841853033,
|
|
"loss": 0.5285,
|
|
"loss/base_kto": 3.074232339859009,
|
|
"metrics/advantage_var": 380.4449462890625,
|
|
"regularization/mmd": 1.1537588834762573,
|
|
"regularization/rkhs_norm": 222.30422973632812,
|
|
"rewards/chosen": 0.673077364580347,
|
|
"rewards/margins": 0.9492099664798078,
|
|
"rewards/rejected": -0.27613260189946087,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 9.622467041015625,
|
|
"kl": 12.925310134887695,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35845182.43902439,
|
|
"logits/rejected": -37073814.974358976,
|
|
"logps/chosen": -488.638671875,
|
|
"logps/rejected": -349.1306590544872,
|
|
"loss": 0.5212,
|
|
"loss/base_kto": 3.0903120040893555,
|
|
"metrics/advantage_var": 329.059326171875,
|
|
"regularization/mmd": 1.0790796279907227,
|
|
"regularization/rkhs_norm": 207.91514587402344,
|
|
"rewards/chosen": 0.5687211199504573,
|
|
"rewards/margins": 0.9100569790046315,
|
|
"rewards/rejected": -0.3413358590541742,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 8.48875617980957,
|
|
"kl": 12.72642707824707,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34771179.06918239,
|
|
"logits/rejected": -36125533.81366459,
|
|
"logps/chosen": -471.2513757861635,
|
|
"logps/rejected": -365.42245729813664,
|
|
"loss": 0.5281,
|
|
"loss/base_kto": 3.0797269344329834,
|
|
"metrics/advantage_var": 402.2024230957031,
|
|
"regularization/mmd": 1.1452440023422241,
|
|
"regularization/rkhs_norm": 220.66357421875,
|
|
"rewards/chosen": 0.5790888948260613,
|
|
"rewards/margins": 0.9379967010329955,
|
|
"rewards/rejected": -0.3589078062069342,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 11.197590827941895,
|
|
"kl": 12.252760887145996,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35302467.637480795,
|
|
"logits/rejected": -36032420.83306836,
|
|
"logps/chosen": -480.44945276497697,
|
|
"logps/rejected": -385.88409181240064,
|
|
"loss": 0.5272,
|
|
"loss/base_kto": 3.123046636581421,
|
|
"metrics/advantage_var": 339.9273376464844,
|
|
"regularization/mmd": 1.0945415496826172,
|
|
"regularization/rkhs_norm": 210.8943328857422,
|
|
"rewards/chosen": 0.5276418940812212,
|
|
"rewards/margins": 0.8905536096872799,
|
|
"rewards/rejected": -0.3629117156060587,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 10.127400398254395,
|
|
"kl": 13.046516418457031,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33633762.78678679,
|
|
"logits/rejected": -36871071.27035831,
|
|
"logps/chosen": -498.67966403903904,
|
|
"logps/rejected": -379.32983000814335,
|
|
"loss": 0.5316,
|
|
"loss/base_kto": 3.103759288787842,
|
|
"metrics/advantage_var": 375.3100280761719,
|
|
"regularization/mmd": 1.1488044261932373,
|
|
"regularization/rkhs_norm": 221.34959411621094,
|
|
"rewards/chosen": 0.6095284126900338,
|
|
"rewards/margins": 0.9271969919904078,
|
|
"rewards/rejected": -0.31766857930037407,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 8.174375534057617,
|
|
"kl": 14.385890007019043,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34574600.05031446,
|
|
"logits/rejected": -36766421.06832298,
|
|
"logps/chosen": -460.4575963050315,
|
|
"logps/rejected": -373.6992915372671,
|
|
"loss": 0.5264,
|
|
"loss/base_kto": 3.0298900604248047,
|
|
"metrics/advantage_var": 427.42755126953125,
|
|
"regularization/mmd": 1.181685447692871,
|
|
"regularization/rkhs_norm": 227.68505859375,
|
|
"rewards/chosen": 0.6774577494687254,
|
|
"rewards/margins": 1.0040204295413915,
|
|
"rewards/rejected": -0.326562680072666,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 9.178069114685059,
|
|
"kl": 14.07157039642334,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34065085.14649682,
|
|
"logits/rejected": -34723500.7607362,
|
|
"logps/chosen": -460.515226910828,
|
|
"logps/rejected": -357.27255080521473,
|
|
"loss": 0.5267,
|
|
"loss/base_kto": 3.108546733856201,
|
|
"metrics/advantage_var": 347.3668518066406,
|
|
"regularization/mmd": 1.1048446893692017,
|
|
"regularization/rkhs_norm": 212.87950134277344,
|
|
"rewards/chosen": 0.5507369314789012,
|
|
"rewards/margins": 0.9131097686625669,
|
|
"rewards/rejected": -0.36237283718366564,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 11.285250663757324,
|
|
"kl": 12.09557056427002,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -36527692.11893584,
|
|
"logits/rejected": -37667141.89079563,
|
|
"logps/chosen": -496.76066118935836,
|
|
"logps/rejected": -353.9441302652106,
|
|
"loss": 0.5238,
|
|
"loss/base_kto": 3.1014554500579834,
|
|
"metrics/advantage_var": 346.4856872558594,
|
|
"regularization/mmd": 1.088848352432251,
|
|
"regularization/rkhs_norm": 209.79734802246094,
|
|
"rewards/chosen": 0.5277840818784233,
|
|
"rewards/margins": 0.905094729113883,
|
|
"rewards/rejected": -0.3773106472354597,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.444851875305176,
|
|
"kl": 15.103259086608887,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -36318720.0,
|
|
"logits/rejected": -36915378.60465116,
|
|
"logps/chosen": -472.03832347972974,
|
|
"logps/rejected": -412.3602834302326,
|
|
"loss": 0.5276,
|
|
"loss/base_kto": 3.086003065109253,
|
|
"metrics/advantage_var": 379.9912414550781,
|
|
"regularization/mmd": 1.134604811668396,
|
|
"regularization/rkhs_norm": 218.6136474609375,
|
|
"rewards/chosen": 0.5286396129711254,
|
|
"rewards/margins": 0.9213921938356103,
|
|
"rewards/rejected": -0.39275258086448495,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 11.337721824645996,
|
|
"kl": 10.847223281860352,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35732298.73291925,
|
|
"logits/rejected": -36615844.2264151,
|
|
"logps/chosen": -485.6144701086956,
|
|
"logps/rejected": -377.01110455974845,
|
|
"loss": 0.526,
|
|
"loss/base_kto": 3.108407974243164,
|
|
"metrics/advantage_var": 344.0582580566406,
|
|
"regularization/mmd": 1.0993179082870483,
|
|
"regularization/rkhs_norm": 211.8146209716797,
|
|
"rewards/chosen": 0.5150987139399748,
|
|
"rewards/margins": 0.9081066281652352,
|
|
"rewards/rejected": -0.39300791422526044,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 10.461943626403809,
|
|
"kl": 12.795968055725098,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35372848.898876406,
|
|
"logits/rejected": -36322177.75342466,
|
|
"logps/chosen": -487.4076043338684,
|
|
"logps/rejected": -386.2137081430746,
|
|
"loss": 0.5227,
|
|
"loss/base_kto": 3.0303561687469482,
|
|
"metrics/advantage_var": 388.92620849609375,
|
|
"regularization/mmd": 1.1511913537979126,
|
|
"regularization/rkhs_norm": 221.8095245361328,
|
|
"rewards/chosen": 0.6047032143484149,
|
|
"rewards/margins": 1.0036851070662611,
|
|
"rewards/rejected": -0.3989818927178463,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 11.451590538024902,
|
|
"kl": 10.240742683410645,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -36673216.29675425,
|
|
"logits/rejected": -37039791.519747235,
|
|
"logps/chosen": -513.1583268933539,
|
|
"logps/rejected": -368.4791172985782,
|
|
"loss": 0.5303,
|
|
"loss/base_kto": 3.0926804542541504,
|
|
"metrics/advantage_var": 380.3329772949219,
|
|
"regularization/mmd": 1.1495593786239624,
|
|
"regularization/rkhs_norm": 221.4950714111328,
|
|
"rewards/chosen": 0.5325430585575252,
|
|
"rewards/margins": 0.9418068260831965,
|
|
"rewards/rejected": -0.4092637675256714,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 12.47182846069336,
|
|
"kl": 12.153326034545898,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -37296432.0,
|
|
"logits/rejected": -36498425.6,
|
|
"logps/chosen": -485.547412109375,
|
|
"logps/rejected": -377.9694580078125,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.0201194286346436,
|
|
"metrics/advantage_var": 419.91796875,
|
|
"regularization/mmd": 1.1779553890228271,
|
|
"regularization/rkhs_norm": 226.9663543701172,
|
|
"rewards/chosen": 0.5866191864013672,
|
|
"rewards/margins": 1.0076179504394531,
|
|
"rewards/rejected": -0.42099876403808595,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 11.72092056274414,
|
|
"kl": 13.146728515625,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35183640.497607656,
|
|
"logits/rejected": -36547529.114854515,
|
|
"logps/chosen": -489.65081738437004,
|
|
"logps/rejected": -365.09939701378255,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.085169553756714,
|
|
"metrics/advantage_var": 342.1353454589844,
|
|
"regularization/mmd": 1.0941046476364136,
|
|
"regularization/rkhs_norm": 210.8101043701172,
|
|
"rewards/chosen": 0.5412538275954446,
|
|
"rewards/margins": 0.9198822881859883,
|
|
"rewards/rejected": -0.37862846059054367,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 15.549859046936035,
|
|
"kl": 10.180974006652832,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -33732301.12302839,
|
|
"logits/rejected": -36279245.2755418,
|
|
"logps/chosen": -458.020011829653,
|
|
"logps/rejected": -383.98679373065016,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.090721368789673,
|
|
"metrics/advantage_var": 371.25335693359375,
|
|
"regularization/mmd": 1.1504759788513184,
|
|
"regularization/rkhs_norm": 221.6716766357422,
|
|
"rewards/chosen": 0.5378277895954505,
|
|
"rewards/margins": 0.9477921473317605,
|
|
"rewards/rejected": -0.40996435773630996,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 12.656947135925293,
|
|
"kl": 11.5225191116333,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -36261612.8,
|
|
"logits/rejected": -36237446.4,
|
|
"logps/chosen": -469.375390625,
|
|
"logps/rejected": -401.649951171875,
|
|
"loss": 0.5229,
|
|
"loss/base_kto": 3.0506389141082764,
|
|
"metrics/advantage_var": 367.6473083496094,
|
|
"regularization/mmd": 1.1327515840530396,
|
|
"regularization/rkhs_norm": 218.25657653808594,
|
|
"rewards/chosen": 0.5057161331176758,
|
|
"rewards/margins": 0.9704075813293458,
|
|
"rewards/rejected": -0.4646914482116699,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 8.844049453735352,
|
|
"kl": 12.70142650604248,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -35099579.07692308,
|
|
"logits/rejected": -36255481.75609756,
|
|
"logps/chosen": -491.3915264423077,
|
|
"logps/rejected": -377.7775342987805,
|
|
"loss": 0.522,
|
|
"loss/base_kto": 3.0711255073547363,
|
|
"metrics/advantage_var": 336.4460754394531,
|
|
"regularization/mmd": 1.1048855781555176,
|
|
"regularization/rkhs_norm": 212.8874053955078,
|
|
"rewards/chosen": 0.5291022276267027,
|
|
"rewards/margins": 0.9443479350688235,
|
|
"rewards/rejected": -0.4152457074421208,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 12.089055061340332,
|
|
"kl": 11.538997650146484,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34441206.33962264,
|
|
"logits/rejected": -36088154.63354037,
|
|
"logps/chosen": -470.83431603773585,
|
|
"logps/rejected": -372.7272175854037,
|
|
"loss": 0.5384,
|
|
"loss/base_kto": 3.090852975845337,
|
|
"metrics/advantage_var": 444.470703125,
|
|
"regularization/mmd": 1.2166645526885986,
|
|
"regularization/rkhs_norm": 234.4247589111328,
|
|
"rewards/chosen": 0.5174438668496953,
|
|
"rewards/margins": 0.9640238414803,
|
|
"rewards/rejected": -0.4465799746306046,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 12.057787895202637,
|
|
"kl": 10.492204666137695,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35121621.13112164,
|
|
"logits/rejected": -35171453.03245749,
|
|
"logps/chosen": -481.2192436808847,
|
|
"logps/rejected": -381.671271251932,
|
|
"loss": 0.5282,
|
|
"loss/base_kto": 3.0740230083465576,
|
|
"metrics/advantage_var": 378.0727233886719,
|
|
"regularization/mmd": 1.1514508724212646,
|
|
"regularization/rkhs_norm": 221.8594970703125,
|
|
"rewards/chosen": 0.5253483439320449,
|
|
"rewards/margins": 0.9607945156410141,
|
|
"rewards/rejected": -0.43544617170896927,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 8.149927139282227,
|
|
"kl": 9.867273330688477,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35843470.39513678,
|
|
"logits/rejected": -34951922.00643087,
|
|
"logps/chosen": -478.296210106383,
|
|
"logps/rejected": -377.64585008038586,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.1066694259643555,
|
|
"metrics/advantage_var": 368.0716247558594,
|
|
"regularization/mmd": 1.1340526342391968,
|
|
"regularization/rkhs_norm": 218.50723266601562,
|
|
"rewards/chosen": 0.511175370143902,
|
|
"rewards/margins": 0.9317512142738508,
|
|
"rewards/rejected": -0.42057584412994875,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 9.785394668579102,
|
|
"kl": 11.317770957946777,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36203450.87598116,
|
|
"logits/rejected": -37434726.32037325,
|
|
"logps/chosen": -493.74921507064363,
|
|
"logps/rejected": -365.62249708398133,
|
|
"loss": 0.5311,
|
|
"loss/base_kto": 3.1075687408447266,
|
|
"metrics/advantage_var": 387.37799072265625,
|
|
"regularization/mmd": 1.1412509679794312,
|
|
"regularization/rkhs_norm": 219.8942108154297,
|
|
"rewards/chosen": 0.5315692608173077,
|
|
"rewards/margins": 0.9077749187288309,
|
|
"rewards/rejected": -0.37620565791152316,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.527176856994629,
|
|
"kl": 11.023845672607422,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -36198295.809220985,
|
|
"logits/rejected": -36452307.95698924,
|
|
"logps/chosen": -480.5086943561208,
|
|
"logps/rejected": -389.5552515360983,
|
|
"loss": 0.5263,
|
|
"loss/base_kto": 3.082139730453491,
|
|
"metrics/advantage_var": 357.8573303222656,
|
|
"regularization/mmd": 1.1284321546554565,
|
|
"regularization/rkhs_norm": 217.42430114746094,
|
|
"rewards/chosen": 0.5544601464688493,
|
|
"rewards/margins": 0.9547959242086247,
|
|
"rewards/rejected": -0.40033577773977536,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 9.05074691772461,
|
|
"kl": 13.44152545928955,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -35885581.385620914,
|
|
"logits/rejected": -36495476.50299401,
|
|
"logps/chosen": -461.20680147058823,
|
|
"logps/rejected": -371.7387022829341,
|
|
"loss": 0.5211,
|
|
"loss/base_kto": 3.0867555141448975,
|
|
"metrics/advantage_var": 348.108154296875,
|
|
"regularization/mmd": 1.0818575620651245,
|
|
"regularization/rkhs_norm": 208.4503936767578,
|
|
"rewards/chosen": 0.5207149530547897,
|
|
"rewards/margins": 0.9165391798453492,
|
|
"rewards/rejected": -0.3958242267905595,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 7.720363616943359,
|
|
"kl": 7.960742473602295,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -35646991.23809524,
|
|
"logits/rejected": -36893005.473684214,
|
|
"logps/chosen": -502.2574869791667,
|
|
"logps/rejected": -359.7631578947368,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.085153579711914,
|
|
"metrics/advantage_var": 348.100341796875,
|
|
"regularization/mmd": 1.1133981943130493,
|
|
"regularization/rkhs_norm": 214.52757263183594,
|
|
"rewards/chosen": 0.5423346019926525,
|
|
"rewards/margins": 0.9594680336782508,
|
|
"rewards/rejected": -0.4171334316855983,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_loss": 0.5584362290272029,
|
|
"train_runtime": 11031.6005,
|
|
"train_samples_per_second": 13.436,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|