a550a8b2fc
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed5 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 20.67009162902832,
|
|
"kl": 1.9479252099990845,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36745789.44,
|
|
"logits/rejected": -38806148.51764706,
|
|
"logps/chosen": -486.13395833333334,
|
|
"logps/rejected": -357.45815716911767,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.873600721359253,
|
|
"metrics/advantage_var": 199.37672424316406,
|
|
"regularization/lipschitz_norm": 943.3322143554688,
|
|
"regularization/mmd": 0.1362994909286499,
|
|
"regularization/rkhs_norm": 162.26129150390625,
|
|
"regularization/w1": 0.7923991084098816,
|
|
"rewards/chosen": -0.2502688344319661,
|
|
"rewards/margins": 0.08248990077598423,
|
|
"rewards/rejected": -0.33275873520795035,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 20.44187355041504,
|
|
"kl": 3.2897727489471436,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35149222.78178964,
|
|
"logits/rejected": -36379245.08864697,
|
|
"logps/chosen": -465.46178375196234,
|
|
"logps/rejected": -379.6661158631415,
|
|
"loss": 0.5941,
|
|
"loss/base_kto": 3.940912961959839,
|
|
"metrics/advantage_var": 152.79051208496094,
|
|
"regularization/lipschitz_norm": 825.8759765625,
|
|
"regularization/mmd": 0.11786098778247833,
|
|
"regularization/rkhs_norm": 140.31069946289062,
|
|
"regularization/w1": 0.6937357783317566,
|
|
"rewards/chosen": -0.023100373883449488,
|
|
"rewards/margins": 0.05192807082496384,
|
|
"rewards/rejected": -0.07502844470841333,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 23.028261184692383,
|
|
"kl": 4.990315914154053,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34687452.12738854,
|
|
"logits/rejected": -37365656.34355828,
|
|
"logps/chosen": -481.92112858280257,
|
|
"logps/rejected": -378.5552386886503,
|
|
"loss": 0.589,
|
|
"loss/base_kto": 3.824028491973877,
|
|
"metrics/advantage_var": 176.4379425048828,
|
|
"regularization/lipschitz_norm": 904.6116333007812,
|
|
"regularization/mmd": 0.12806551158428192,
|
|
"regularization/rkhs_norm": 152.4589385986328,
|
|
"regularization/w1": 0.7598738074302673,
|
|
"rewards/chosen": 0.024187677225489524,
|
|
"rewards/margins": 0.1632834282799374,
|
|
"rewards/rejected": -0.13909575105444785,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 28.615266799926758,
|
|
"kl": 3.786818027496338,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36571955.2,
|
|
"logits/rejected": -37887108.885496184,
|
|
"logps/chosen": -479.1769,
|
|
"logps/rejected": -371.84217557251907,
|
|
"loss": 0.5936,
|
|
"loss/base_kto": 3.852421522140503,
|
|
"metrics/advantage_var": 198.41836547851562,
|
|
"regularization/lipschitz_norm": 910.3705444335938,
|
|
"regularization/mmd": 0.131294384598732,
|
|
"regularization/rkhs_norm": 156.30284118652344,
|
|
"regularization/w1": 0.7647112607955933,
|
|
"rewards/chosen": 0.05660130615234375,
|
|
"rewards/margins": 0.15614959940437143,
|
|
"rewards/rejected": -0.09954829325202767,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 24.30677032470703,
|
|
"kl": 4.0709686279296875,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35430678.820097245,
|
|
"logits/rejected": -36955505.13423831,
|
|
"logps/chosen": -448.11841572123177,
|
|
"logps/rejected": -361.8265460030166,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.8806111812591553,
|
|
"metrics/advantage_var": 162.788818359375,
|
|
"regularization/lipschitz_norm": 857.3562622070312,
|
|
"regularization/mmd": 0.12275510281324387,
|
|
"regularization/rkhs_norm": 146.13702392578125,
|
|
"regularization/w1": 0.7201792597770691,
|
|
"rewards/chosen": -0.018179986233456208,
|
|
"rewards/margins": 0.09848384731566356,
|
|
"rewards/rejected": -0.11666383354911977,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 27.274593353271484,
|
|
"kl": 16.874574661254883,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35657038.30588235,
|
|
"logits/rejected": -36616512.85333333,
|
|
"logps/chosen": -481.6936580882353,
|
|
"logps/rejected": -367.100703125,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.8359532356262207,
|
|
"metrics/advantage_var": 202.48439025878906,
|
|
"regularization/lipschitz_norm": 964.8781127929688,
|
|
"regularization/mmd": 0.13669872283935547,
|
|
"regularization/rkhs_norm": 162.736572265625,
|
|
"regularization/w1": 0.8104975819587708,
|
|
"rewards/chosen": 0.23281494589412913,
|
|
"rewards/margins": 0.15118915370866365,
|
|
"rewards/rejected": 0.08162579218546549,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 24.10140609741211,
|
|
"kl": 21.570547103881836,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35325692.17910448,
|
|
"logits/rejected": -36794008.76065574,
|
|
"logps/chosen": -473.5366604477612,
|
|
"logps/rejected": -354.75676229508196,
|
|
"loss": 0.5916,
|
|
"loss/base_kto": 3.7875802516937256,
|
|
"metrics/advantage_var": 201.50961303710938,
|
|
"regularization/lipschitz_norm": 960.6087036132812,
|
|
"regularization/mmd": 0.13858595490455627,
|
|
"regularization/rkhs_norm": 164.9832763671875,
|
|
"regularization/w1": 0.8069114089012146,
|
|
"rewards/chosen": 0.3974429002448694,
|
|
"rewards/margins": 0.18101103549077102,
|
|
"rewards/rejected": 0.21643186475409837,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 23.85628890991211,
|
|
"kl": 17.967758178710938,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -37210617.559748426,
|
|
"logits/rejected": -37033786.83229814,
|
|
"logps/chosen": -473.3713639937107,
|
|
"logps/rejected": -363.1606414984472,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.7869834899902344,
|
|
"metrics/advantage_var": 201.8460693359375,
|
|
"regularization/lipschitz_norm": 949.5863647460938,
|
|
"regularization/mmd": 0.13735289871692657,
|
|
"regularization/rkhs_norm": 163.51536560058594,
|
|
"regularization/w1": 0.7976524233818054,
|
|
"rewards/chosen": 0.28570477467662886,
|
|
"rewards/margins": 0.18095441983870184,
|
|
"rewards/rejected": 0.10475035483792702,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.733516693115234,
|
|
"kl": 10.516373634338379,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36012218.474097334,
|
|
"logits/rejected": -37267675.76982893,
|
|
"logps/chosen": -471.9190541601256,
|
|
"logps/rejected": -348.91874027993777,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.753783941268921,
|
|
"metrics/advantage_var": 225.03857421875,
|
|
"regularization/lipschitz_norm": 1012.8804931640625,
|
|
"regularization/mmd": 0.14312028884887695,
|
|
"regularization/rkhs_norm": 170.3813018798828,
|
|
"regularization/w1": 0.8508195877075195,
|
|
"rewards/chosen": 0.1454004141000601,
|
|
"rewards/margins": 0.2274882675629082,
|
|
"rewards/rejected": -0.0820878534628481,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 22.378005981445312,
|
|
"kl": 19.50136375427246,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37822526.62996942,
|
|
"logits/rejected": -37933265.38019169,
|
|
"logps/chosen": -508.591121941896,
|
|
"logps/rejected": -361.4126397763578,
|
|
"loss": 0.5949,
|
|
"loss/base_kto": 3.7431771755218506,
|
|
"metrics/advantage_var": 230.7646484375,
|
|
"regularization/lipschitz_norm": 1031.614501953125,
|
|
"regularization/mmd": 0.1491633653640747,
|
|
"regularization/rkhs_norm": 177.57542419433594,
|
|
"regularization/w1": 0.8665561676025391,
|
|
"rewards/chosen": 0.36003075462597955,
|
|
"rewards/margins": 0.2266431006535854,
|
|
"rewards/rejected": 0.13338765397239416,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 20.391454696655273,
|
|
"kl": 20.642642974853516,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37195620.61305007,
|
|
"logits/rejected": -39158485.53945249,
|
|
"logps/chosen": -496.6452010622155,
|
|
"logps/rejected": -375.17811493558776,
|
|
"loss": 0.5898,
|
|
"loss/base_kto": 3.7457730770111084,
|
|
"metrics/advantage_var": 228.0854949951172,
|
|
"regularization/lipschitz_norm": 986.2400512695312,
|
|
"regularization/mmd": 0.14456598460674286,
|
|
"regularization/rkhs_norm": 172.1023712158203,
|
|
"regularization/w1": 0.8284416198730469,
|
|
"rewards/chosen": 0.4145056763621491,
|
|
"rewards/margins": 0.23657980478798066,
|
|
"rewards/rejected": 0.17792587157416842,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 21.531587600708008,
|
|
"kl": 14.1281099319458,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36354458.874028,
|
|
"logits/rejected": -37320385.70800628,
|
|
"logps/chosen": -474.1392398911353,
|
|
"logps/rejected": -365.2935145211931,
|
|
"loss": 0.5919,
|
|
"loss/base_kto": 3.8021271228790283,
|
|
"metrics/advantage_var": 209.7309112548828,
|
|
"regularization/lipschitz_norm": 947.1066284179688,
|
|
"regularization/mmd": 0.13743311166763306,
|
|
"regularization/rkhs_norm": 163.61083984375,
|
|
"regularization/w1": 0.7955695986747742,
|
|
"rewards/chosen": 0.25722177366075766,
|
|
"rewards/margins": 0.17277734736823042,
|
|
"rewards/rejected": 0.08444442629252723,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 18.736751556396484,
|
|
"kl": 6.519759654998779,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36864680.51653543,
|
|
"logits/rejected": -38052208.32248062,
|
|
"logps/chosen": -488.0936023622047,
|
|
"logps/rejected": -380.5171027131783,
|
|
"loss": 0.5995,
|
|
"loss/base_kto": 3.8403961658477783,
|
|
"metrics/advantage_var": 209.14273071289062,
|
|
"regularization/lipschitz_norm": 970.310546875,
|
|
"regularization/mmd": 0.1402624249458313,
|
|
"regularization/rkhs_norm": 166.9790802001953,
|
|
"regularization/w1": 0.8150609135627747,
|
|
"rewards/chosen": 0.04636677419106791,
|
|
"rewards/margins": 0.1527477277210764,
|
|
"rewards/rejected": -0.10638095353000848,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 24.319786071777344,
|
|
"kl": 9.462249755859375,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -34828143.8208,
|
|
"logits/rejected": -37237870.99847328,
|
|
"logps/chosen": -478.8565,
|
|
"logps/rejected": -384.2642891221374,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.741943359375,
|
|
"metrics/advantage_var": 268.43017578125,
|
|
"regularization/lipschitz_norm": 1053.147705078125,
|
|
"regularization/mmd": 0.15234512090682983,
|
|
"regularization/rkhs_norm": 181.36326599121094,
|
|
"regularization/w1": 0.8846439719200134,
|
|
"rewards/chosen": 0.118431494140625,
|
|
"rewards/margins": 0.21270740985433562,
|
|
"rewards/rejected": -0.09427591571371063,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 23.534250259399414,
|
|
"kl": 11.045422554016113,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36356810.27160494,
|
|
"logits/rejected": -37444384.405063294,
|
|
"logps/chosen": -481.43745177469134,
|
|
"logps/rejected": -390.0705597310127,
|
|
"loss": 0.5972,
|
|
"loss/base_kto": 3.782033681869507,
|
|
"metrics/advantage_var": 228.792724609375,
|
|
"regularization/lipschitz_norm": 1012.0086059570312,
|
|
"regularization/mmd": 0.145858034491539,
|
|
"regularization/rkhs_norm": 173.64051818847656,
|
|
"regularization/w1": 0.8500871658325195,
|
|
"rewards/chosen": 0.21624364970642843,
|
|
"rewards/margins": 0.19835357059443587,
|
|
"rewards/rejected": 0.017890079111992557,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 19.872377395629883,
|
|
"kl": 23.93448829650879,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -38243406.532330826,
|
|
"logits/rejected": -38514947.74634147,
|
|
"logps/chosen": -468.2688439849624,
|
|
"logps/rejected": -386.70543699186993,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.7427849769592285,
|
|
"metrics/advantage_var": 245.9417724609375,
|
|
"regularization/lipschitz_norm": 1043.07763671875,
|
|
"regularization/mmd": 0.15340642631053925,
|
|
"regularization/rkhs_norm": 182.62669372558594,
|
|
"regularization/w1": 0.8761852383613586,
|
|
"rewards/chosen": 0.4155246361753994,
|
|
"rewards/margins": 0.23698629666907955,
|
|
"rewards/rejected": 0.17853833950631987,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 22.400741577148438,
|
|
"kl": 24.455322265625,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -37996048.77379095,
|
|
"logits/rejected": -37879176.61345853,
|
|
"logps/chosen": -507.0338826053042,
|
|
"logps/rejected": -363.8024990219092,
|
|
"loss": 0.5936,
|
|
"loss/base_kto": 3.7253081798553467,
|
|
"metrics/advantage_var": 256.9432067871094,
|
|
"regularization/lipschitz_norm": 1035.1435546875,
|
|
"regularization/mmd": 0.1540052592754364,
|
|
"regularization/rkhs_norm": 183.339599609375,
|
|
"regularization/w1": 0.8695206046104431,
|
|
"rewards/chosen": 0.42766895978573516,
|
|
"rewards/margins": 0.23152582288817794,
|
|
"rewards/rejected": 0.19614313689755722,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 22.976598739624023,
|
|
"kl": 7.938958644866943,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35306468.324324325,
|
|
"logits/rejected": -36213476.48208469,
|
|
"logps/chosen": -490.83436561561564,
|
|
"logps/rejected": -389.009161237785,
|
|
"loss": 0.6019,
|
|
"loss/base_kto": 3.7460715770721436,
|
|
"metrics/advantage_var": 253.7123565673828,
|
|
"regularization/lipschitz_norm": 1091.4351806640625,
|
|
"regularization/mmd": 0.15219855308532715,
|
|
"regularization/rkhs_norm": 181.18875122070312,
|
|
"regularization/w1": 0.9168054461479187,
|
|
"rewards/chosen": 0.16097977784302858,
|
|
"rewards/margins": 0.25080174554887014,
|
|
"rewards/rejected": -0.08982196770584157,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 20.255399703979492,
|
|
"kl": 13.6774263381958,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -34702110.21118012,
|
|
"logits/rejected": -37870836.72955975,
|
|
"logps/chosen": -495.23325892857144,
|
|
"logps/rejected": -356.2152122641509,
|
|
"loss": 0.5919,
|
|
"loss/base_kto": 3.7645881175994873,
|
|
"metrics/advantage_var": 229.5393829345703,
|
|
"regularization/lipschitz_norm": 985.5276489257812,
|
|
"regularization/mmd": 0.14242756366729736,
|
|
"regularization/rkhs_norm": 169.55662536621094,
|
|
"regularization/w1": 0.8278433084487915,
|
|
"rewards/chosen": 0.24982983133067255,
|
|
"rewards/margins": 0.19894242958191935,
|
|
"rewards/rejected": 0.0508874017487532,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 21.399656295776367,
|
|
"kl": 15.086886405944824,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36604398.45482866,
|
|
"logits/rejected": -38395666.45768025,
|
|
"logps/chosen": -477.8134735202492,
|
|
"logps/rejected": -379.2145131269593,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.763843536376953,
|
|
"metrics/advantage_var": 235.6248016357422,
|
|
"regularization/lipschitz_norm": 1036.3138427734375,
|
|
"regularization/mmd": 0.1487146019935608,
|
|
"regularization/rkhs_norm": 177.0412139892578,
|
|
"regularization/w1": 0.8705036044120789,
|
|
"rewards/chosen": 0.27532802118319216,
|
|
"rewards/margins": 0.23370120455021282,
|
|
"rewards/rejected": 0.04162681663297934,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 23.87718391418457,
|
|
"kl": 25.548213958740234,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36724739.16049383,
|
|
"logits/rejected": -37107731.44303797,
|
|
"logps/chosen": -483.24546682098764,
|
|
"logps/rejected": -380.68052808544303,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.77905535697937,
|
|
"metrics/advantage_var": 258.1446228027344,
|
|
"regularization/lipschitz_norm": 1038.5205078125,
|
|
"regularization/mmd": 0.14911852777004242,
|
|
"regularization/rkhs_norm": 177.5220489501953,
|
|
"regularization/w1": 0.872357189655304,
|
|
"rewards/chosen": 0.3292643935592086,
|
|
"rewards/margins": 0.15456709703033203,
|
|
"rewards/rejected": 0.17469729652887658,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 21.920976638793945,
|
|
"kl": 12.123173713684082,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37376329.48895899,
|
|
"logits/rejected": -38672032.099071205,
|
|
"logps/chosen": -485.65605283911674,
|
|
"logps/rejected": -413.11745356037153,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.794027805328369,
|
|
"metrics/advantage_var": 270.994873046875,
|
|
"regularization/lipschitz_norm": 1040.5550537109375,
|
|
"regularization/mmd": 0.14962628483772278,
|
|
"regularization/rkhs_norm": 178.1265411376953,
|
|
"regularization/w1": 0.874066174030304,
|
|
"rewards/chosen": 0.15646629453833547,
|
|
"rewards/margins": 0.1721192389673374,
|
|
"rewards/rejected": -0.01565294442900194,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 19.79609489440918,
|
|
"kl": 8.25532341003418,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -34981651.0681458,
|
|
"logits/rejected": -37423933.929121725,
|
|
"logps/chosen": -502.71374801901743,
|
|
"logps/rejected": -369.4180469953775,
|
|
"loss": 0.5999,
|
|
"loss/base_kto": 3.7118279933929443,
|
|
"metrics/advantage_var": 274.7864685058594,
|
|
"regularization/lipschitz_norm": 1107.482421875,
|
|
"regularization/mmd": 0.15676438808441162,
|
|
"regularization/rkhs_norm": 186.624267578125,
|
|
"regularization/w1": 0.9302851557731628,
|
|
"rewards/chosen": 0.08697658484409049,
|
|
"rewards/margins": 0.281008780747945,
|
|
"rewards/rejected": -0.19403219590385448,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 22.302595138549805,
|
|
"kl": 19.128345489501953,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -38235444.43373494,
|
|
"logits/rejected": -38718304.415584415,
|
|
"logps/chosen": -484.83819653614455,
|
|
"logps/rejected": -376.224609375,
|
|
"loss": 0.6068,
|
|
"loss/base_kto": 3.7533974647521973,
|
|
"metrics/advantage_var": 289.1072692871094,
|
|
"regularization/lipschitz_norm": 1117.0125732421875,
|
|
"regularization/mmd": 0.16270272433757782,
|
|
"regularization/rkhs_norm": 193.69371032714844,
|
|
"regularization/w1": 0.9382905960083008,
|
|
"rewards/chosen": 0.27394630248288077,
|
|
"rewards/margins": 0.20642831593756572,
|
|
"rewards/rejected": 0.06751798654531503,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 19.231454849243164,
|
|
"kl": 14.655878067016602,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -37233006.05238829,
|
|
"logits/rejected": -39298224.88748019,
|
|
"logps/chosen": -507.7286691063174,
|
|
"logps/rejected": -371.590778526149,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.765949249267578,
|
|
"metrics/advantage_var": 252.63059997558594,
|
|
"regularization/lipschitz_norm": 1098.7313232421875,
|
|
"regularization/mmd": 0.154534250497818,
|
|
"regularization/rkhs_norm": 183.96934509277344,
|
|
"regularization/w1": 0.922934353351593,
|
|
"rewards/chosen": 0.2643245938011604,
|
|
"rewards/margins": 0.2025455353114894,
|
|
"rewards/rejected": 0.06177905848967103,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 26.971399307250977,
|
|
"kl": 16.3587703704834,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -37068427.03089431,
|
|
"logits/rejected": -39943499.067669176,
|
|
"logps/chosen": -503.4192581300813,
|
|
"logps/rejected": -344.97246240601504,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.69382905960083,
|
|
"metrics/advantage_var": 247.6055145263672,
|
|
"regularization/lipschitz_norm": 1020.49658203125,
|
|
"regularization/mmd": 0.14904962480068207,
|
|
"regularization/rkhs_norm": 177.4400177001953,
|
|
"regularization/w1": 0.857217013835907,
|
|
"rewards/chosen": 0.28179301440231197,
|
|
"rewards/margins": 0.25828919603375877,
|
|
"rewards/rejected": 0.02350381836855322,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 19.392044067382812,
|
|
"kl": 10.862930297851562,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36843257.35572519,
|
|
"logits/rejected": -37337851.4944,
|
|
"logps/chosen": -469.96450381679387,
|
|
"logps/rejected": -394.654925,
|
|
"loss": 0.5931,
|
|
"loss/base_kto": 3.7576234340667725,
|
|
"metrics/advantage_var": 233.93408203125,
|
|
"regularization/lipschitz_norm": 1001.2546997070312,
|
|
"regularization/mmd": 0.1461522877216339,
|
|
"regularization/rkhs_norm": 173.99082946777344,
|
|
"regularization/w1": 0.8410539031028748,
|
|
"rewards/chosen": 0.11657686888716604,
|
|
"rewards/margins": 0.22866212279341602,
|
|
"rewards/rejected": -0.11208525390625,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.202499389648438,
|
|
"kl": 17.498945236206055,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36128863.948303714,
|
|
"logits/rejected": -38197582.620272316,
|
|
"logps/chosen": -482.4040791599354,
|
|
"logps/rejected": -361.5468986384266,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.7312629222869873,
|
|
"metrics/advantage_var": 259.32977294921875,
|
|
"regularization/lipschitz_norm": 1032.8582763671875,
|
|
"regularization/mmd": 0.15501855313777924,
|
|
"regularization/rkhs_norm": 184.54588317871094,
|
|
"regularization/w1": 0.8676008582115173,
|
|
"rewards/chosen": 0.2711190514880351,
|
|
"rewards/margins": 0.22723794824942292,
|
|
"rewards/rejected": 0.04388110323861219,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 20.515392303466797,
|
|
"kl": 8.992721557617188,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35947380.81553398,
|
|
"logits/rejected": -37067819.311178245,
|
|
"logps/chosen": -481.2120246763754,
|
|
"logps/rejected": -357.7751841012085,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.7053017616271973,
|
|
"metrics/advantage_var": 268.7295837402344,
|
|
"regularization/lipschitz_norm": 1055.6337890625,
|
|
"regularization/mmd": 0.15584371984004974,
|
|
"regularization/rkhs_norm": 185.5282440185547,
|
|
"regularization/w1": 0.8867325186729431,
|
|
"rewards/chosen": 0.16149211007028721,
|
|
"rewards/margins": 0.2782246353915792,
|
|
"rewards/rejected": -0.11673252532129201,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 16.0169734954834,
|
|
"kl": 16.430936813354492,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36692030.62996942,
|
|
"logits/rejected": -37772641.32907348,
|
|
"logps/chosen": -486.86066513761466,
|
|
"logps/rejected": -382.26255491214056,
|
|
"loss": 0.6038,
|
|
"loss/base_kto": 3.7231247425079346,
|
|
"metrics/advantage_var": 280.35968017578125,
|
|
"regularization/lipschitz_norm": 1126.2333984375,
|
|
"regularization/mmd": 0.1615525484085083,
|
|
"regularization/rkhs_norm": 192.324462890625,
|
|
"regularization/w1": 0.9460359811782837,
|
|
"rewards/chosen": 0.22247624761832235,
|
|
"rewards/margins": 0.2452185647449042,
|
|
"rewards/rejected": -0.022742317126581844,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 21.753538131713867,
|
|
"kl": 14.277803421020508,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -36299428.03883495,
|
|
"logits/rejected": -37487204.54380665,
|
|
"logps/chosen": -495.65544093851133,
|
|
"logps/rejected": -384.2934762084592,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.7704811096191406,
|
|
"metrics/advantage_var": 226.02835083007812,
|
|
"regularization/lipschitz_norm": 1014.4932861328125,
|
|
"regularization/mmd": 0.14600291848182678,
|
|
"regularization/rkhs_norm": 173.81300354003906,
|
|
"regularization/w1": 0.8521742820739746,
|
|
"rewards/chosen": 0.21826060767312652,
|
|
"rewards/margins": 0.20829108220625367,
|
|
"rewards/rejected": 0.009969525466872846,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 22.93177604675293,
|
|
"kl": 11.957457542419434,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -36371445.08980213,
|
|
"logits/rejected": -37876290.5682183,
|
|
"logps/chosen": -463.1000761035008,
|
|
"logps/rejected": -372.7319923756019,
|
|
"loss": 0.595,
|
|
"loss/base_kto": 3.7308413982391357,
|
|
"metrics/advantage_var": 244.42764282226562,
|
|
"regularization/lipschitz_norm": 1046.0361328125,
|
|
"regularization/mmd": 0.1500902771949768,
|
|
"regularization/rkhs_norm": 178.67889404296875,
|
|
"regularization/w1": 0.878670334815979,
|
|
"rewards/chosen": 0.22403820392022214,
|
|
"rewards/margins": 0.25638787153824044,
|
|
"rewards/rejected": -0.03234966761801828,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 24.102664947509766,
|
|
"kl": 17.31543731689453,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36464313.82542113,
|
|
"logits/rejected": -37198408.67623605,
|
|
"logps/chosen": -495.7117151607963,
|
|
"logps/rejected": -364.50396232057415,
|
|
"loss": 0.5911,
|
|
"loss/base_kto": 3.725515842437744,
|
|
"metrics/advantage_var": 232.67544555664062,
|
|
"regularization/lipschitz_norm": 1020.2478637695312,
|
|
"regularization/mmd": 0.14654068648815155,
|
|
"regularization/rkhs_norm": 174.45323181152344,
|
|
"regularization/w1": 0.857008159160614,
|
|
"rewards/chosen": 0.2674788610858537,
|
|
"rewards/margins": 0.2269748435708081,
|
|
"rewards/rejected": 0.04050401751504561,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 24.527793884277344,
|
|
"kl": 14.906573295593262,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36681850.364779875,
|
|
"logits/rejected": -36880317.217391305,
|
|
"logps/chosen": -484.5278105345912,
|
|
"logps/rejected": -365.1701766304348,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.710954427719116,
|
|
"metrics/advantage_var": 260.28436279296875,
|
|
"regularization/lipschitz_norm": 1097.6351318359375,
|
|
"regularization/mmd": 0.15554527938365936,
|
|
"regularization/rkhs_norm": 185.17295837402344,
|
|
"regularization/w1": 0.9220134615898132,
|
|
"rewards/chosen": 0.24328354169737618,
|
|
"rewards/margins": 0.26278783707019104,
|
|
"rewards/rejected": -0.019504295372814867,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 20.30431365966797,
|
|
"kl": 12.334071159362793,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37734572.60291734,
|
|
"logits/rejected": -38451607.746606335,
|
|
"logps/chosen": -489.3760129659643,
|
|
"logps/rejected": -388.30649038461536,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.74849009513855,
|
|
"metrics/advantage_var": 264.3041687011719,
|
|
"regularization/lipschitz_norm": 1038.9027099609375,
|
|
"regularization/mmd": 0.15424896776676178,
|
|
"regularization/rkhs_norm": 183.62974548339844,
|
|
"regularization/w1": 0.8726783990859985,
|
|
"rewards/chosen": 0.18337045728290874,
|
|
"rewards/margins": 0.23592790627850757,
|
|
"rewards/rejected": -0.05255744899559884,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 20.224668502807617,
|
|
"kl": 12.596447944641113,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -34952888.32,
|
|
"logits/rejected": -38083996.85079365,
|
|
"logps/chosen": -461.76548076923075,
|
|
"logps/rejected": -380.7762648809524,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.7214317321777344,
|
|
"metrics/advantage_var": 251.2515411376953,
|
|
"regularization/lipschitz_norm": 1064.1939697265625,
|
|
"regularization/mmd": 0.1525294929742813,
|
|
"regularization/rkhs_norm": 181.58274841308594,
|
|
"regularization/w1": 0.8939228057861328,
|
|
"rewards/chosen": 0.2392426476111779,
|
|
"rewards/margins": 0.22293902507777325,
|
|
"rewards/rejected": 0.016303622533404637,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.575621604919434,
|
|
"kl": 12.5536470413208,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36143697.668711655,
|
|
"logits/rejected": -37686258.95541401,
|
|
"logps/chosen": -497.0175421779141,
|
|
"logps/rejected": -362.5546875,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.752284288406372,
|
|
"metrics/advantage_var": 297.0749206542969,
|
|
"regularization/lipschitz_norm": 1117.095947265625,
|
|
"regularization/mmd": 0.15982839465141296,
|
|
"regularization/rkhs_norm": 190.2718963623047,
|
|
"regularization/w1": 0.9383605122566223,
|
|
"rewards/chosen": 0.16839938953610287,
|
|
"rewards/margins": 0.2048152246311622,
|
|
"rewards/rejected": -0.03641583509505934,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 17.124082565307617,
|
|
"kl": 15.74393367767334,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35181453.48460292,
|
|
"logits/rejected": -36652404.223227754,
|
|
"logps/chosen": -499.7367807941653,
|
|
"logps/rejected": -373.7562688536953,
|
|
"loss": 0.6016,
|
|
"loss/base_kto": 3.7982919216156006,
|
|
"metrics/advantage_var": 254.0915069580078,
|
|
"regularization/lipschitz_norm": 1026.367431640625,
|
|
"regularization/mmd": 0.15220218896865845,
|
|
"regularization/rkhs_norm": 181.19308471679688,
|
|
"regularization/w1": 0.8621487021446228,
|
|
"rewards/chosen": 0.17538462040683245,
|
|
"rewards/margins": 0.1723997816190459,
|
|
"rewards/rejected": 0.0029848387877865613,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 17.833799362182617,
|
|
"kl": 4.535059452056885,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -35980446.55483871,
|
|
"logits/rejected": -36960458.31003039,
|
|
"logps/chosen": -518.8235383064516,
|
|
"logps/rejected": -378.08826462765956,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.5379321575164795,
|
|
"metrics/advantage_var": 435.5148620605469,
|
|
"regularization/lipschitz_norm": 1313.1517333984375,
|
|
"regularization/mmd": 0.1887841373682022,
|
|
"regularization/rkhs_norm": 224.74302673339844,
|
|
"regularization/w1": 1.1030473709106445,
|
|
"rewards/chosen": 0.04880948220529864,
|
|
"rewards/margins": 0.5075058881436297,
|
|
"rewards/rejected": -0.4586964059383311,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 21.87369155883789,
|
|
"kl": 10.168845176696777,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36591457.96535433,
|
|
"logits/rejected": -37067441.01705427,
|
|
"logps/chosen": -481.8382874015748,
|
|
"logps/rejected": -377.7716812015504,
|
|
"loss": 0.6138,
|
|
"loss/base_kto": 3.3144783973693848,
|
|
"metrics/advantage_var": 589.0369262695312,
|
|
"regularization/lipschitz_norm": 1633.3592529296875,
|
|
"regularization/mmd": 0.2240116149187088,
|
|
"regularization/rkhs_norm": 266.6805114746094,
|
|
"regularization/w1": 1.3720217943191528,
|
|
"rewards/chosen": 0.3213261641855315,
|
|
"rewards/margins": 0.7587058918078669,
|
|
"rewards/rejected": -0.4373797276223353,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 18.56248664855957,
|
|
"kl": 9.8152437210083,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35298182.981818184,
|
|
"logits/rejected": -36804135.63870968,
|
|
"logps/chosen": -480.85700757575756,
|
|
"logps/rejected": -368.7930191532258,
|
|
"loss": 0.6239,
|
|
"loss/base_kto": 3.3169960975646973,
|
|
"metrics/advantage_var": 661.1056518554688,
|
|
"regularization/lipschitz_norm": 1710.257080078125,
|
|
"regularization/mmd": 0.2378852665424347,
|
|
"regularization/rkhs_norm": 283.1967468261719,
|
|
"regularization/w1": 1.4366158246994019,
|
|
"rewards/chosen": 0.4595697576349432,
|
|
"rewards/margins": 0.7583705644803314,
|
|
"rewards/rejected": -0.2988008068453881,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 17.172359466552734,
|
|
"kl": 13.928451538085938,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35261304.44657097,
|
|
"logits/rejected": -36561234.71975498,
|
|
"logps/chosen": -478.76305821371614,
|
|
"logps/rejected": -420.7439222817764,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.201690435409546,
|
|
"metrics/advantage_var": 621.3167114257812,
|
|
"regularization/lipschitz_norm": 1605.9063720703125,
|
|
"regularization/mmd": 0.2283145934343338,
|
|
"regularization/rkhs_norm": 271.8031311035156,
|
|
"regularization/w1": 1.3489612340927124,
|
|
"rewards/chosen": 0.48136758271967206,
|
|
"rewards/margins": 0.8656706771366658,
|
|
"rewards/rejected": -0.3843030944169937,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 16.447967529296875,
|
|
"kl": 9.176518440246582,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35890979.74959872,
|
|
"logits/rejected": -36695830.210045666,
|
|
"logps/chosen": -460.6535413322632,
|
|
"logps/rejected": -381.8759512937595,
|
|
"loss": 0.6182,
|
|
"loss/base_kto": 3.335076093673706,
|
|
"metrics/advantage_var": 640.2711791992188,
|
|
"regularization/lipschitz_norm": 1644.6497802734375,
|
|
"regularization/mmd": 0.22922053933143616,
|
|
"regularization/rkhs_norm": 272.8816223144531,
|
|
"regularization/w1": 1.3815058469772339,
|
|
"rewards/chosen": 0.3886109867983798,
|
|
"rewards/margins": 0.7655763048946627,
|
|
"rewards/rejected": -0.3769653180962828,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 23.55685043334961,
|
|
"kl": 12.506660461425781,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -36198894.98489426,
|
|
"logits/rejected": -37032605.41100324,
|
|
"logps/chosen": -475.5877549093656,
|
|
"logps/rejected": -378.5860891990291,
|
|
"loss": 0.6044,
|
|
"loss/base_kto": 3.284658908843994,
|
|
"metrics/advantage_var": 581.3097534179688,
|
|
"regularization/lipschitz_norm": 1581.3773193359375,
|
|
"regularization/mmd": 0.2217867225408554,
|
|
"regularization/rkhs_norm": 264.0318298339844,
|
|
"regularization/w1": 1.3283569812774658,
|
|
"rewards/chosen": 0.45500657925793053,
|
|
"rewards/margins": 0.8361634512634549,
|
|
"rewards/rejected": -0.3811568720055244,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 15.991052627563477,
|
|
"kl": 21.35040283203125,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35681537.18335901,
|
|
"logits/rejected": -36956020.43740095,
|
|
"logps/chosen": -478.8996533127889,
|
|
"logps/rejected": -365.395750792393,
|
|
"loss": 0.6029,
|
|
"loss/base_kto": 3.2986061573028564,
|
|
"metrics/advantage_var": 582.2473754882812,
|
|
"regularization/lipschitz_norm": 1549.0313720703125,
|
|
"regularization/mmd": 0.22335056960582733,
|
|
"regularization/rkhs_norm": 265.8935241699219,
|
|
"regularization/w1": 1.301186442375183,
|
|
"rewards/chosen": 0.6604523034235362,
|
|
"rewards/margins": 0.7518704835718468,
|
|
"rewards/rejected": -0.09141818014831059,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 21.807615280151367,
|
|
"kl": 18.00168800354004,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -36559937.9089482,
|
|
"logits/rejected": -38227885.1881804,
|
|
"logps/chosen": -484.48910910518055,
|
|
"logps/rejected": -371.1508067651633,
|
|
"loss": 0.6135,
|
|
"loss/base_kto": 3.2756271362304688,
|
|
"metrics/advantage_var": 726.1383056640625,
|
|
"regularization/lipschitz_norm": 1659.885986328125,
|
|
"regularization/mmd": 0.23783467710018158,
|
|
"regularization/rkhs_norm": 283.13653564453125,
|
|
"regularization/w1": 1.3943042755126953,
|
|
"rewards/chosen": 0.5675269206240189,
|
|
"rewards/margins": 0.7931678657026299,
|
|
"rewards/rejected": -0.22564094507861102,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 21.79051399230957,
|
|
"kl": 13.512042045593262,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35249636.54258675,
|
|
"logits/rejected": -35610585.956656344,
|
|
"logps/chosen": -491.02070189274446,
|
|
"logps/rejected": -375.1051905959752,
|
|
"loss": 0.6036,
|
|
"loss/base_kto": 3.2885327339172363,
|
|
"metrics/advantage_var": 611.0409545898438,
|
|
"regularization/lipschitz_norm": 1565.328125,
|
|
"regularization/mmd": 0.22567521035671234,
|
|
"regularization/rkhs_norm": 268.66094970703125,
|
|
"regularization/w1": 1.314875602722168,
|
|
"rewards/chosen": 0.4869826644753056,
|
|
"rewards/margins": 0.7988680376174181,
|
|
"rewards/rejected": -0.3118853731421125,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 13.28946304321289,
|
|
"kl": 14.684597969055176,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -36197448.112676054,
|
|
"logits/rejected": -36856261.691107646,
|
|
"logps/chosen": -447.47378716744913,
|
|
"logps/rejected": -371.0314937597504,
|
|
"loss": 0.5862,
|
|
"loss/base_kto": 3.239898681640625,
|
|
"metrics/advantage_var": 551.99609375,
|
|
"regularization/lipschitz_norm": 1472.745849609375,
|
|
"regularization/mmd": 0.21260903775691986,
|
|
"regularization/rkhs_norm": 253.1060028076172,
|
|
"regularization/w1": 1.2371065616607666,
|
|
"rewards/chosen": 0.4805610191094484,
|
|
"rewards/margins": 0.8032315138198618,
|
|
"rewards/rejected": -0.3226704947104134,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 19.637243270874023,
|
|
"kl": 9.189848899841309,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34937800.20512821,
|
|
"logits/rejected": -36114091.70731708,
|
|
"logps/chosen": -498.53881209935895,
|
|
"logps/rejected": -369.2965415396341,
|
|
"loss": 0.6079,
|
|
"loss/base_kto": 3.259673833847046,
|
|
"metrics/advantage_var": 614.0714721679688,
|
|
"regularization/lipschitz_norm": 1637.0792236328125,
|
|
"regularization/mmd": 0.2280350774526596,
|
|
"regularization/rkhs_norm": 271.4703063964844,
|
|
"regularization/w1": 1.3751466274261475,
|
|
"rewards/chosen": 0.4148457845052083,
|
|
"rewards/margins": 0.8178668758733485,
|
|
"rewards/rejected": -0.40302109136814024,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 15.831979751586914,
|
|
"kl": 15.795271873474121,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -36102406.87761194,
|
|
"logits/rejected": -37408576.6295082,
|
|
"logps/chosen": -496.81539179104476,
|
|
"logps/rejected": -389.46357581967214,
|
|
"loss": 0.6159,
|
|
"loss/base_kto": 3.332146406173706,
|
|
"metrics/advantage_var": 589.1945190429688,
|
|
"regularization/lipschitz_norm": 1625.651123046875,
|
|
"regularization/mmd": 0.2293902486562729,
|
|
"regularization/rkhs_norm": 273.0836181640625,
|
|
"regularization/w1": 1.3655469417572021,
|
|
"rewards/chosen": 0.4604987756529851,
|
|
"rewards/margins": 0.6943137780943913,
|
|
"rewards/rejected": -0.23381500244140624,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 18.846372604370117,
|
|
"kl": 6.967624187469482,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34727523.863777086,
|
|
"logits/rejected": -36741740.214511044,
|
|
"logps/chosen": -470.32594814241486,
|
|
"logps/rejected": -366.38194992113563,
|
|
"loss": 0.6102,
|
|
"loss/base_kto": 3.3769073486328125,
|
|
"metrics/advantage_var": 544.1055297851562,
|
|
"regularization/lipschitz_norm": 1533.817626953125,
|
|
"regularization/mmd": 0.21603214740753174,
|
|
"regularization/rkhs_norm": 257.18115234375,
|
|
"regularization/w1": 1.2884068489074707,
|
|
"rewards/chosen": 0.34355999736963044,
|
|
"rewards/margins": 0.7039021254133015,
|
|
"rewards/rejected": -0.36034212804367116,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 16.535308837890625,
|
|
"kl": 11.569666862487793,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35818812.50909091,
|
|
"logits/rejected": -35605583.27741936,
|
|
"logps/chosen": -502.7956439393939,
|
|
"logps/rejected": -366.6118699596774,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.2230117321014404,
|
|
"metrics/advantage_var": 632.7213745117188,
|
|
"regularization/lipschitz_norm": 1622.147705078125,
|
|
"regularization/mmd": 0.23166914284229279,
|
|
"regularization/rkhs_norm": 275.7966003417969,
|
|
"regularization/w1": 1.362604022026062,
|
|
"rewards/chosen": 0.5035967740145597,
|
|
"rewards/margins": 0.8731916187096203,
|
|
"rewards/rejected": -0.3695948446950605,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 17.559795379638672,
|
|
"kl": 17.243452072143555,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35458056.03767661,
|
|
"logits/rejected": -37407341.08864697,
|
|
"logps/chosen": -506.42602040816325,
|
|
"logps/rejected": -372.56738433125975,
|
|
"loss": 0.6172,
|
|
"loss/base_kto": 3.2942306995391846,
|
|
"metrics/advantage_var": 618.4035034179688,
|
|
"regularization/lipschitz_norm": 1680.7093505859375,
|
|
"regularization/mmd": 0.2311849147081375,
|
|
"regularization/rkhs_norm": 275.2201843261719,
|
|
"regularization/w1": 1.411795973777771,
|
|
"rewards/chosen": 0.49288413438542483,
|
|
"rewards/margins": 0.7521722043759478,
|
|
"rewards/rejected": -0.25928806999052295,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 21.55283546447754,
|
|
"kl": 18.124265670776367,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34883935.357032456,
|
|
"logits/rejected": -36409874.60347551,
|
|
"logps/chosen": -470.66190108191654,
|
|
"logps/rejected": -372.9338467614534,
|
|
"loss": 0.604,
|
|
"loss/base_kto": 3.367394208908081,
|
|
"metrics/advantage_var": 511.4122009277344,
|
|
"regularization/lipschitz_norm": 1487.292236328125,
|
|
"regularization/mmd": 0.21556456387043,
|
|
"regularization/rkhs_norm": 256.6244812011719,
|
|
"regularization/w1": 1.2493253946304321,
|
|
"rewards/chosen": 0.5155438241855439,
|
|
"rewards/margins": 0.6709385847271736,
|
|
"rewards/rejected": -0.15539476054162965,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 17.61018180847168,
|
|
"kl": 11.695985794067383,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34725701.2327044,
|
|
"logits/rejected": -35829683.677018635,
|
|
"logps/chosen": -466.3694968553459,
|
|
"logps/rejected": -373.62533967391306,
|
|
"loss": 0.602,
|
|
"loss/base_kto": 3.3153557777404785,
|
|
"metrics/advantage_var": 524.4066772460938,
|
|
"regularization/lipschitz_norm": 1529.5323486328125,
|
|
"regularization/mmd": 0.21546566486358643,
|
|
"regularization/rkhs_norm": 256.5067443847656,
|
|
"regularization/w1": 1.2848070859909058,
|
|
"rewards/chosen": 0.5077054965421088,
|
|
"rewards/margins": 0.7631978550163664,
|
|
"rewards/rejected": -0.25549235847425755,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 17.635663986206055,
|
|
"kl": 9.056909561157227,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34020453.59370079,
|
|
"logits/rejected": -35876978.30697674,
|
|
"logps/chosen": -477.1708661417323,
|
|
"logps/rejected": -380.3998546511628,
|
|
"loss": 0.6159,
|
|
"loss/base_kto": 3.348447561264038,
|
|
"metrics/advantage_var": 632.1333618164062,
|
|
"regularization/lipschitz_norm": 1609.167724609375,
|
|
"regularization/mmd": 0.22681903839111328,
|
|
"regularization/rkhs_norm": 270.0226745605469,
|
|
"regularization/w1": 1.351700782775879,
|
|
"rewards/chosen": 0.30686888206662155,
|
|
"rewards/margins": 0.7083278588713696,
|
|
"rewards/rejected": -0.40145897680474807,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.189739227294922,
|
|
"kl": 10.588912010192871,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34492528.855753645,
|
|
"logits/rejected": -37875242.47360483,
|
|
"logps/chosen": -480.3084987844408,
|
|
"logps/rejected": -364.04859539969834,
|
|
"loss": 0.6032,
|
|
"loss/base_kto": 3.29600191116333,
|
|
"metrics/advantage_var": 570.6554565429688,
|
|
"regularization/lipschitz_norm": 1555.6051025390625,
|
|
"regularization/mmd": 0.22321109473705292,
|
|
"regularization/rkhs_norm": 265.7275085449219,
|
|
"regularization/w1": 1.3067082166671753,
|
|
"rewards/chosen": 0.4437416113756078,
|
|
"rewards/margins": 0.7693087964452486,
|
|
"rewards/rejected": -0.32556718506964083,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 18.128437042236328,
|
|
"kl": 10.108545303344727,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34681704.97244733,
|
|
"logits/rejected": -35432420.199095026,
|
|
"logps/chosen": -490.55287682333875,
|
|
"logps/rejected": -372.83887160633486,
|
|
"loss": 0.6072,
|
|
"loss/base_kto": 3.3541133403778076,
|
|
"metrics/advantage_var": 590.4301147460938,
|
|
"regularization/lipschitz_norm": 1529.4583740234375,
|
|
"regularization/mmd": 0.21881555020809174,
|
|
"regularization/rkhs_norm": 260.4947204589844,
|
|
"regularization/w1": 1.2847449779510498,
|
|
"rewards/chosen": 0.3427481628121201,
|
|
"rewards/margins": 0.7054281520817162,
|
|
"rewards/rejected": -0.36267998926959605,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 17.019926071166992,
|
|
"kl": 16.55487060546875,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34213762.35582822,
|
|
"logits/rejected": -36385994.191082805,
|
|
"logps/chosen": -488.32563266871165,
|
|
"logps/rejected": -383.2771198248408,
|
|
"loss": 0.6231,
|
|
"loss/base_kto": 3.2450668811798096,
|
|
"metrics/advantage_var": 761.9019775390625,
|
|
"regularization/lipschitz_norm": 1777.282470703125,
|
|
"regularization/mmd": 0.24660912156105042,
|
|
"regularization/rkhs_norm": 293.582275390625,
|
|
"regularization/w1": 1.4929174184799194,
|
|
"rewards/chosen": 0.5532671220463478,
|
|
"rewards/margins": 0.8092904561364652,
|
|
"rewards/rejected": -0.25602333409011746,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 16.47247886657715,
|
|
"kl": 11.24480152130127,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34316371.330203444,
|
|
"logits/rejected": -35128161.847113885,
|
|
"logps/chosen": -472.56685250391234,
|
|
"logps/rejected": -385.1658053822153,
|
|
"loss": 0.6102,
|
|
"loss/base_kto": 3.301698684692383,
|
|
"metrics/advantage_var": 624.9652709960938,
|
|
"regularization/lipschitz_norm": 1611.0435791015625,
|
|
"regularization/mmd": 0.22677765786647797,
|
|
"regularization/rkhs_norm": 269.973388671875,
|
|
"regularization/w1": 1.3532766103744507,
|
|
"rewards/chosen": 0.4035864219605829,
|
|
"rewards/margins": 0.7785279815126134,
|
|
"rewards/rejected": -0.3749415595520305,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 15.375174522399902,
|
|
"kl": 12.943961143493652,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35163236.13880126,
|
|
"logits/rejected": -35130776.965944275,
|
|
"logps/chosen": -472.6810922712934,
|
|
"logps/rejected": -411.40097716718265,
|
|
"loss": 0.6017,
|
|
"loss/base_kto": 3.2502992153167725,
|
|
"metrics/advantage_var": 719.5261840820312,
|
|
"regularization/lipschitz_norm": 1584.6982421875,
|
|
"regularization/mmd": 0.23204405605793,
|
|
"regularization/rkhs_norm": 276.242919921875,
|
|
"regularization/w1": 1.3311465978622437,
|
|
"rewards/chosen": 0.4705423060275779,
|
|
"rewards/margins": 0.8415792016850494,
|
|
"rewards/rejected": -0.37103689565747144,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 16.864608764648438,
|
|
"kl": 7.824089050292969,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34898280.013266996,
|
|
"logits/rejected": -36188247.72821271,
|
|
"logps/chosen": -496.92283374792703,
|
|
"logps/rejected": -352.1043436115214,
|
|
"loss": 0.6152,
|
|
"loss/base_kto": 3.3539161682128906,
|
|
"metrics/advantage_var": 675.998046875,
|
|
"regularization/lipschitz_norm": 1597.838623046875,
|
|
"regularization/mmd": 0.22515200078487396,
|
|
"regularization/rkhs_norm": 268.0380859375,
|
|
"regularization/w1": 1.3421844244003296,
|
|
"rewards/chosen": 0.2688575099356732,
|
|
"rewards/margins": 0.7027325445553187,
|
|
"rewards/rejected": -0.4338750346196455,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.610519409179688,
|
|
"kl": 6.846913814544678,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -33535580.455235206,
|
|
"logits/rejected": -32836286.45410628,
|
|
"logps/chosen": -507.3798842943854,
|
|
"logps/rejected": -412.5739231078905,
|
|
"loss": 0.6081,
|
|
"loss/base_kto": 3.2501351833343506,
|
|
"metrics/advantage_var": 642.9050903320312,
|
|
"regularization/lipschitz_norm": 1646.7279052734375,
|
|
"regularization/mmd": 0.23111307621002197,
|
|
"regularization/rkhs_norm": 275.1346130371094,
|
|
"regularization/w1": 1.383251428604126,
|
|
"rewards/chosen": 0.3652233071681762,
|
|
"rewards/margins": 0.8627621455777474,
|
|
"rewards/rejected": -0.49753883840957125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 17.609745025634766,
|
|
"kl": 11.589434623718262,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34552121.83254345,
|
|
"logits/rejected": -35316326.083462134,
|
|
"logps/chosen": -495.454285150079,
|
|
"logps/rejected": -384.5259611669243,
|
|
"loss": 0.6162,
|
|
"loss/base_kto": 3.3512799739837646,
|
|
"metrics/advantage_var": 615.2280883789062,
|
|
"regularization/lipschitz_norm": 1615.4227294921875,
|
|
"regularization/mmd": 0.22117102146148682,
|
|
"regularization/rkhs_norm": 263.298828125,
|
|
"regularization/w1": 1.3569550514221191,
|
|
"rewards/chosen": 0.3657852100535027,
|
|
"rewards/margins": 0.6919076531804347,
|
|
"rewards/rejected": -0.326122443126932,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 13.600857734680176,
|
|
"kl": 10.581927299499512,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33810810.56969697,
|
|
"logits/rejected": -36186072.36129032,
|
|
"logps/chosen": -499.5164772727273,
|
|
"logps/rejected": -359.4179435483871,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.3308751583099365,
|
|
"metrics/advantage_var": 489.8287658691406,
|
|
"regularization/lipschitz_norm": 1476.373291015625,
|
|
"regularization/mmd": 0.2084968090057373,
|
|
"regularization/rkhs_norm": 248.2104949951172,
|
|
"regularization/w1": 1.2401536703109741,
|
|
"rewards/chosen": 0.42881303267045456,
|
|
"rewards/margins": 0.7129002322199758,
|
|
"rewards/rejected": -0.28408719954952116,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 17.946428298950195,
|
|
"kl": 16.50902557373047,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -31569818.55108359,
|
|
"logits/rejected": -35038372.74447949,
|
|
"logps/chosen": -502.62393575851394,
|
|
"logps/rejected": -358.6391462933754,
|
|
"loss": 0.6113,
|
|
"loss/base_kto": 3.308062791824341,
|
|
"metrics/advantage_var": 860.5252075195312,
|
|
"regularization/lipschitz_norm": 1611.06982421875,
|
|
"regularization/mmd": 0.22933049499988556,
|
|
"regularization/rkhs_norm": 273.01251220703125,
|
|
"regularization/w1": 1.353298544883728,
|
|
"rewards/chosen": 0.5563234028063322,
|
|
"rewards/margins": 0.7589931662209739,
|
|
"rewards/rejected": -0.20266976341464166,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 14.805122375488281,
|
|
"kl": 13.904166221618652,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -32885823.015384614,
|
|
"logits/rejected": -35195933.25714286,
|
|
"logps/chosen": -487.61908653846154,
|
|
"logps/rejected": -365.45689484126984,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.298060655593872,
|
|
"metrics/advantage_var": 564.461181640625,
|
|
"regularization/lipschitz_norm": 1547.9945068359375,
|
|
"regularization/mmd": 0.2226027101278305,
|
|
"regularization/rkhs_norm": 265.00323486328125,
|
|
"regularization/w1": 1.300315260887146,
|
|
"rewards/chosen": 0.534131845327524,
|
|
"rewards/margins": 0.7471982128250438,
|
|
"rewards/rejected": -0.21306636749751984,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 14.087535858154297,
|
|
"kl": 15.207712173461914,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34511047.962204725,
|
|
"logits/rejected": -35748240.074418604,
|
|
"logps/chosen": -475.345374015748,
|
|
"logps/rejected": -374.0406492248062,
|
|
"loss": 0.5866,
|
|
"loss/base_kto": 3.3016624450683594,
|
|
"metrics/advantage_var": 463.8214416503906,
|
|
"regularization/lipschitz_norm": 1413.26171875,
|
|
"regularization/mmd": 0.2043001651763916,
|
|
"regularization/rkhs_norm": 243.2144775390625,
|
|
"regularization/w1": 1.187139868736267,
|
|
"rewards/chosen": 0.4673630121186024,
|
|
"rewards/margins": 0.7458367168484958,
|
|
"rewards/rejected": -0.2784737047298934,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 22.939298629760742,
|
|
"kl": 10.584595680236816,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34324027.015576325,
|
|
"logits/rejected": -34701870.54545455,
|
|
"logps/chosen": -486.6362441588785,
|
|
"logps/rejected": -360.1907327586207,
|
|
"loss": 0.595,
|
|
"loss/base_kto": 3.322956085205078,
|
|
"metrics/advantage_var": 530.6657104492188,
|
|
"regularization/lipschitz_norm": 1459.2164306640625,
|
|
"regularization/mmd": 0.21153569221496582,
|
|
"regularization/rkhs_norm": 251.82823181152344,
|
|
"regularization/w1": 1.2257417440414429,
|
|
"rewards/chosen": 0.36884842631972836,
|
|
"rewards/margins": 0.7581468205019875,
|
|
"rewards/rejected": -0.38929839418225903,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 14.420248031616211,
|
|
"kl": 11.56763744354248,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34694611.26213592,
|
|
"logits/rejected": -35103759.468277946,
|
|
"logps/chosen": -484.4076152912621,
|
|
"logps/rejected": -388.3403984138973,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.2739503383636475,
|
|
"metrics/advantage_var": 488.2797546386719,
|
|
"regularization/lipschitz_norm": 1472.9439697265625,
|
|
"regularization/mmd": 0.2099769562482834,
|
|
"regularization/rkhs_norm": 249.97256469726562,
|
|
"regularization/w1": 1.23727285861969,
|
|
"rewards/chosen": 0.4295048883431938,
|
|
"rewards/margins": 0.7899368245553704,
|
|
"rewards/rejected": -0.3604319362121766,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 15.317728042602539,
|
|
"kl": 11.60511302947998,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33141669.78670788,
|
|
"logits/rejected": -32971547.90521327,
|
|
"logps/chosen": -456.81196870170015,
|
|
"logps/rejected": -382.5088368878357,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.2631123065948486,
|
|
"metrics/advantage_var": 560.9888916015625,
|
|
"regularization/lipschitz_norm": 1459.6904296875,
|
|
"regularization/mmd": 0.21361985802650452,
|
|
"regularization/rkhs_norm": 254.3093719482422,
|
|
"regularization/w1": 1.2261399030685425,
|
|
"rewards/chosen": 0.3788637046283327,
|
|
"rewards/margins": 0.8087955380974283,
|
|
"rewards/rejected": -0.4299318334690956,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 25.00137710571289,
|
|
"kl": 15.943984985351562,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34763131.37833595,
|
|
"logits/rejected": -36049622.19595645,
|
|
"logps/chosen": -491.8460557299843,
|
|
"logps/rejected": -377.21875,
|
|
"loss": 0.5996,
|
|
"loss/base_kto": 3.297459125518799,
|
|
"metrics/advantage_var": 555.6985473632812,
|
|
"regularization/lipschitz_norm": 1525.255126953125,
|
|
"regularization/mmd": 0.21810658276081085,
|
|
"regularization/rkhs_norm": 259.6507263183594,
|
|
"regularization/w1": 1.2812143564224243,
|
|
"rewards/chosen": 0.5720050039246468,
|
|
"rewards/margins": 0.7381607107782019,
|
|
"rewards/rejected": -0.1661557068535551,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 16.53514862060547,
|
|
"kl": 11.54581356048584,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34169040.0,
|
|
"logits/rejected": -35373100.8,
|
|
"logps/chosen": -479.863671875,
|
|
"logps/rejected": -374.3803466796875,
|
|
"loss": 0.604,
|
|
"loss/base_kto": 3.3393285274505615,
|
|
"metrics/advantage_var": 592.9227905273438,
|
|
"regularization/lipschitz_norm": 1516.8388671875,
|
|
"regularization/mmd": 0.2185019999742508,
|
|
"regularization/rkhs_norm": 260.1214294433594,
|
|
"regularization/w1": 1.2741446495056152,
|
|
"rewards/chosen": 0.3619662284851074,
|
|
"rewards/margins": 0.6888193607330322,
|
|
"rewards/rejected": -0.3268531322479248,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 19.912586212158203,
|
|
"kl": 13.483344078063965,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35517263.01234568,
|
|
"logits/rejected": -35183846.07594936,
|
|
"logps/chosen": -463.275993441358,
|
|
"logps/rejected": -370.4110957278481,
|
|
"loss": 0.5966,
|
|
"loss/base_kto": 3.306652069091797,
|
|
"metrics/advantage_var": 530.9874877929688,
|
|
"regularization/lipschitz_norm": 1489.7181396484375,
|
|
"regularization/mmd": 0.2145804613828659,
|
|
"regularization/rkhs_norm": 255.4529266357422,
|
|
"regularization/w1": 1.2513631582260132,
|
|
"rewards/chosen": 0.4715083086932147,
|
|
"rewards/margins": 0.7539580551269074,
|
|
"rewards/rejected": -0.28244974643369264,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 15.422378540039062,
|
|
"kl": 16.968021392822266,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -33524504.975609757,
|
|
"logits/rejected": -34687527.384615384,
|
|
"logps/chosen": -473.8185022865854,
|
|
"logps/rejected": -375.02183493589746,
|
|
"loss": 0.5938,
|
|
"loss/base_kto": 3.32004714012146,
|
|
"metrics/advantage_var": 483.416015625,
|
|
"regularization/lipschitz_norm": 1461.2935791015625,
|
|
"regularization/mmd": 0.20300164818763733,
|
|
"regularization/rkhs_norm": 241.66860961914062,
|
|
"regularization/w1": 1.2274866104125977,
|
|
"rewards/chosen": 0.5240411991026344,
|
|
"rewards/margins": 0.7307511831239435,
|
|
"rewards/rejected": -0.2067099840213091,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 16.63542366027832,
|
|
"kl": 13.394705772399902,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34061198.222222224,
|
|
"logits/rejected": -36468310.23975721,
|
|
"logps/chosen": -497.9873691626409,
|
|
"logps/rejected": -356.5102902124431,
|
|
"loss": 0.5982,
|
|
"loss/base_kto": 3.2797813415527344,
|
|
"metrics/advantage_var": 549.18701171875,
|
|
"regularization/lipschitz_norm": 1534.291015625,
|
|
"regularization/mmd": 0.21697306632995605,
|
|
"regularization/rkhs_norm": 258.30126953125,
|
|
"regularization/w1": 1.2888044118881226,
|
|
"rewards/chosen": 0.46995792603915054,
|
|
"rewards/margins": 0.7730949802509652,
|
|
"rewards/rejected": -0.30313705421181475,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 16.30461883544922,
|
|
"kl": 11.491045951843262,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33323437.827160493,
|
|
"logits/rejected": -34646132.658227846,
|
|
"logps/chosen": -442.68113425925924,
|
|
"logps/rejected": -374.8646904667722,
|
|
"loss": 0.6121,
|
|
"loss/base_kto": 3.3129944801330566,
|
|
"metrics/advantage_var": 655.145751953125,
|
|
"regularization/lipschitz_norm": 1613.0155029296875,
|
|
"regularization/mmd": 0.22887134552001953,
|
|
"regularization/rkhs_norm": 272.4659118652344,
|
|
"regularization/w1": 1.3549330234527588,
|
|
"rewards/chosen": 0.42668396749614196,
|
|
"rewards/margins": 0.7510802457809149,
|
|
"rewards/rejected": -0.324396278284773,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 16.893821716308594,
|
|
"kl": 13.179420471191406,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34527845.732899025,
|
|
"logits/rejected": -34666079.61445783,
|
|
"logps/chosen": -450.8351995114007,
|
|
"logps/rejected": -354.81240587349396,
|
|
"loss": 0.5771,
|
|
"loss/base_kto": 3.2603137493133545,
|
|
"metrics/advantage_var": 471.96337890625,
|
|
"regularization/lipschitz_norm": 1374.1279296875,
|
|
"regularization/mmd": 0.20227418839931488,
|
|
"regularization/rkhs_norm": 240.80259704589844,
|
|
"regularization/w1": 1.1542673110961914,
|
|
"rewards/chosen": 0.48127850958112783,
|
|
"rewards/margins": 0.7877240564781887,
|
|
"rewards/rejected": -0.3064455468970609,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 17.633975982666016,
|
|
"kl": 9.367804527282715,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34220206.63565891,
|
|
"logits/rejected": -35114175.8992126,
|
|
"logps/chosen": -482.91913759689925,
|
|
"logps/rejected": -387.0054625984252,
|
|
"loss": 0.568,
|
|
"loss/base_kto": 3.1954779624938965,
|
|
"metrics/advantage_var": 469.1932678222656,
|
|
"regularization/lipschitz_norm": 1364.8748779296875,
|
|
"regularization/mmd": 0.20240406692028046,
|
|
"regularization/rkhs_norm": 240.9572296142578,
|
|
"regularization/w1": 1.1464948654174805,
|
|
"rewards/chosen": 0.40537838011749033,
|
|
"rewards/margins": 0.8599942719420475,
|
|
"rewards/rejected": -0.4546158918245571,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.641488075256348,
|
|
"kl": 12.354438781738281,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34263179.78412698,
|
|
"logits/rejected": -36049595.470769234,
|
|
"logps/chosen": -486.7924107142857,
|
|
"logps/rejected": -356.91990384615383,
|
|
"loss": 0.5675,
|
|
"loss/base_kto": 3.1480367183685303,
|
|
"metrics/advantage_var": 484.9521484375,
|
|
"regularization/lipschitz_norm": 1406.8016357421875,
|
|
"regularization/mmd": 0.20991837978363037,
|
|
"regularization/rkhs_norm": 249.90283203125,
|
|
"regularization/w1": 1.181713342666626,
|
|
"rewards/chosen": 0.5818535698784723,
|
|
"rewards/margins": 0.9040121955545539,
|
|
"rewards/rejected": -0.3221586256760817,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 13.56850528717041,
|
|
"kl": 6.684207439422607,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34272953.965996906,
|
|
"logits/rejected": -35289689.78199052,
|
|
"logps/chosen": -466.5400888717156,
|
|
"logps/rejected": -386.13534261453395,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.2365081310272217,
|
|
"metrics/advantage_var": 398.34033203125,
|
|
"regularization/lipschitz_norm": 1295.4384765625,
|
|
"regularization/mmd": 0.19008252024650574,
|
|
"regularization/rkhs_norm": 226.2886962890625,
|
|
"regularization/w1": 1.0881683826446533,
|
|
"rewards/chosen": 0.3401842544767919,
|
|
"rewards/margins": 0.8140668965026114,
|
|
"rewards/rejected": -0.47388264202581953,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 14.188789367675781,
|
|
"kl": 9.632389068603516,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34290703.28358209,
|
|
"logits/rejected": -35077727.6852459,
|
|
"logps/chosen": -487.1331623134328,
|
|
"logps/rejected": -387.50548155737704,
|
|
"loss": 0.5685,
|
|
"loss/base_kto": 3.240670919418335,
|
|
"metrics/advantage_var": 411.7604064941406,
|
|
"regularization/lipschitz_norm": 1329.864990234375,
|
|
"regularization/mmd": 0.19018827378749847,
|
|
"regularization/rkhs_norm": 226.4146270751953,
|
|
"regularization/w1": 1.11708664894104,
|
|
"rewards/chosen": 0.41139394276177704,
|
|
"rewards/margins": 0.816758682852069,
|
|
"rewards/rejected": -0.405364740090292,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 12.198801040649414,
|
|
"kl": 17.952430725097656,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -32834949.31629393,
|
|
"logits/rejected": -35845780.74617737,
|
|
"logps/chosen": -466.93140974440894,
|
|
"logps/rejected": -365.73184250764524,
|
|
"loss": 0.5581,
|
|
"loss/base_kto": 3.1707189083099365,
|
|
"metrics/advantage_var": 428.1657409667969,
|
|
"regularization/lipschitz_norm": 1308.3392333984375,
|
|
"regularization/mmd": 0.195432648062706,
|
|
"regularization/rkhs_norm": 232.65794372558594,
|
|
"regularization/w1": 1.0990049839019775,
|
|
"rewards/chosen": 0.5989690908608726,
|
|
"rewards/margins": 0.8394078496409758,
|
|
"rewards/rejected": -0.2404387587801032,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.00127410888672,
|
|
"kl": 12.2958402633667,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34211327.24148148,
|
|
"logits/rejected": -34209643.054545455,
|
|
"logps/chosen": -487.9811111111111,
|
|
"logps/rejected": -378.4548811983471,
|
|
"loss": 0.5609,
|
|
"loss/base_kto": 3.2125489711761475,
|
|
"metrics/advantage_var": 404.812744140625,
|
|
"regularization/lipschitz_norm": 1289.4254150390625,
|
|
"regularization/mmd": 0.19118943810462952,
|
|
"regularization/rkhs_norm": 227.60647583007812,
|
|
"regularization/w1": 1.0831173658370972,
|
|
"rewards/chosen": 0.5194717520254629,
|
|
"rewards/margins": 0.811769594508353,
|
|
"rewards/rejected": -0.29229784248289,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 18.6423282623291,
|
|
"kl": 17.075681686401367,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34602311.29113924,
|
|
"logits/rejected": -35104764.83950617,
|
|
"logps/chosen": -485.4895174050633,
|
|
"logps/rejected": -370.2487461419753,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.145045280456543,
|
|
"metrics/advantage_var": 455.00244140625,
|
|
"regularization/lipschitz_norm": 1302.3179931640625,
|
|
"regularization/mmd": 0.19558389484882355,
|
|
"regularization/rkhs_norm": 232.8379364013672,
|
|
"regularization/w1": 1.0939470529556274,
|
|
"rewards/chosen": 0.5698854470554786,
|
|
"rewards/margins": 0.8855800440878137,
|
|
"rewards/rejected": -0.31569459703233504,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 19.949146270751953,
|
|
"kl": 12.951250076293945,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34040358.25755167,
|
|
"logits/rejected": -36859237.06298003,
|
|
"logps/chosen": -496.67299284578695,
|
|
"logps/rejected": -351.168946812596,
|
|
"loss": 0.5526,
|
|
"loss/base_kto": 3.169118642807007,
|
|
"metrics/advantage_var": 390.81341552734375,
|
|
"regularization/lipschitz_norm": 1265.3895263671875,
|
|
"regularization/mmd": 0.18876340985298157,
|
|
"regularization/rkhs_norm": 224.7183380126953,
|
|
"regularization/w1": 1.06292724609375,
|
|
"rewards/chosen": 0.5121310526691674,
|
|
"rewards/margins": 0.8567477331367173,
|
|
"rewards/rejected": -0.3446166804675499,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 17.865863800048828,
|
|
"kl": 13.589856147766113,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33942589.15773354,
|
|
"logits/rejected": -34251989.94577353,
|
|
"logps/chosen": -480.28589203675347,
|
|
"logps/rejected": -387.3678478867624,
|
|
"loss": 0.5584,
|
|
"loss/base_kto": 3.1252965927124023,
|
|
"metrics/advantage_var": 466.70489501953125,
|
|
"regularization/lipschitz_norm": 1357.9537353515625,
|
|
"regularization/mmd": 0.2011280059814453,
|
|
"regularization/rkhs_norm": 239.43809509277344,
|
|
"regularization/w1": 1.1406811475753784,
|
|
"rewards/chosen": 0.5978173902900077,
|
|
"rewards/margins": 0.926751617170241,
|
|
"rewards/rejected": -0.32893422688023327,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 16.44749641418457,
|
|
"kl": 16.37274169921875,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -33290848.0,
|
|
"logits/rejected": -34991280.0,
|
|
"logps/chosen": -503.111181640625,
|
|
"logps/rejected": -400.1039794921875,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.1718239784240723,
|
|
"metrics/advantage_var": 457.9571228027344,
|
|
"regularization/lipschitz_norm": 1362.3905029296875,
|
|
"regularization/mmd": 0.2020179033279419,
|
|
"regularization/rkhs_norm": 240.4975128173828,
|
|
"regularization/w1": 1.1444079875946045,
|
|
"rewards/chosen": 0.5759394645690918,
|
|
"rewards/margins": 0.86002836227417,
|
|
"rewards/rejected": -0.2840888977050781,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 19.371198654174805,
|
|
"kl": 12.099383354187012,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34977602.85350318,
|
|
"logits/rejected": -35392822.969325155,
|
|
"logps/chosen": -496.6204219745223,
|
|
"logps/rejected": -370.08934049079755,
|
|
"loss": 0.5502,
|
|
"loss/base_kto": 3.127540349960327,
|
|
"metrics/advantage_var": 432.763916015625,
|
|
"regularization/lipschitz_norm": 1285.8685302734375,
|
|
"regularization/mmd": 0.19408564269542694,
|
|
"regularization/rkhs_norm": 231.05435180664062,
|
|
"regularization/w1": 1.080129623413086,
|
|
"rewards/chosen": 0.5249679468239948,
|
|
"rewards/margins": 0.904853637899232,
|
|
"rewards/rejected": -0.37988569107523723,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 14.959543228149414,
|
|
"kl": 12.483757019042969,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -33898518.5408805,
|
|
"logits/rejected": -35877837.11801242,
|
|
"logps/chosen": -480.4602004716981,
|
|
"logps/rejected": -372.8230298913044,
|
|
"loss": 0.5484,
|
|
"loss/base_kto": 3.1473729610443115,
|
|
"metrics/advantage_var": 383.9928894042969,
|
|
"regularization/lipschitz_norm": 1251.9901123046875,
|
|
"regularization/mmd": 0.18830907344818115,
|
|
"regularization/rkhs_norm": 224.17747497558594,
|
|
"regularization/w1": 1.0516717433929443,
|
|
"rewards/chosen": 0.539827694682955,
|
|
"rewards/margins": 0.878365112266539,
|
|
"rewards/rejected": -0.338537417583584,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 16.395963668823242,
|
|
"kl": 12.685641288757324,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33862333.59869494,
|
|
"logits/rejected": -35913586.75862069,
|
|
"logps/chosen": -463.567088091354,
|
|
"logps/rejected": -380.15744471514245,
|
|
"loss": 0.552,
|
|
"loss/base_kto": 3.1507604122161865,
|
|
"metrics/advantage_var": 395.13922119140625,
|
|
"regularization/lipschitz_norm": 1282.7816162109375,
|
|
"regularization/mmd": 0.1873190850019455,
|
|
"regularization/rkhs_norm": 222.99893188476562,
|
|
"regularization/w1": 1.0775364637374878,
|
|
"rewards/chosen": 0.4674279864810614,
|
|
"rewards/margins": 0.8704747846151974,
|
|
"rewards/rejected": -0.4030467981341361,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 16.599842071533203,
|
|
"kl": 12.55467700958252,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33685516.60307692,
|
|
"logits/rejected": -35014642.9968254,
|
|
"logps/chosen": -477.19615384615383,
|
|
"logps/rejected": -369.03978174603174,
|
|
"loss": 0.5615,
|
|
"loss/base_kto": 3.199857711791992,
|
|
"metrics/advantage_var": 425.13525390625,
|
|
"regularization/lipschitz_norm": 1306.302001953125,
|
|
"regularization/mmd": 0.1948663741350174,
|
|
"regularization/rkhs_norm": 231.98379516601562,
|
|
"regularization/w1": 1.097293734550476,
|
|
"rewards/chosen": 0.4951728703425481,
|
|
"rewards/margins": 0.8536986097398696,
|
|
"rewards/rejected": -0.3585257393973214,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 15.497091293334961,
|
|
"kl": 12.510932922363281,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33772782.4238806,
|
|
"logits/rejected": -36386779.06885246,
|
|
"logps/chosen": -481.1972014925373,
|
|
"logps/rejected": -392.2569159836066,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.202043294906616,
|
|
"metrics/advantage_var": 396.09527587890625,
|
|
"regularization/lipschitz_norm": 1263.4398193359375,
|
|
"regularization/mmd": 0.18781693279743195,
|
|
"regularization/rkhs_norm": 223.59158325195312,
|
|
"regularization/w1": 1.061289668083191,
|
|
"rewards/chosen": 0.5571805583896922,
|
|
"rewards/margins": 0.8350556894651917,
|
|
"rewards/rejected": -0.2778751310754995,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 14.99384593963623,
|
|
"kl": 17.907392501831055,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33640813.598736174,
|
|
"logits/rejected": -34056157.18083462,
|
|
"logps/chosen": -498.21929304897316,
|
|
"logps/rejected": -372.19914026275114,
|
|
"loss": 0.5502,
|
|
"loss/base_kto": 3.11967134475708,
|
|
"metrics/advantage_var": 407.9004211425781,
|
|
"regularization/lipschitz_norm": 1296.3050537109375,
|
|
"regularization/mmd": 0.19289694726467133,
|
|
"regularization/rkhs_norm": 229.6392059326172,
|
|
"regularization/w1": 1.08889639377594,
|
|
"rewards/chosen": 0.5916010900319658,
|
|
"rewards/margins": 0.8917609040322798,
|
|
"rewards/rejected": -0.30015981400031394,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 14.467615127563477,
|
|
"kl": 12.251875877380371,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34850161.9910045,
|
|
"logits/rejected": -34718998.96900489,
|
|
"logps/chosen": -486.04375937031483,
|
|
"logps/rejected": -388.5187347063622,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.199481964111328,
|
|
"metrics/advantage_var": 437.51171875,
|
|
"regularization/lipschitz_norm": 1291.8138427734375,
|
|
"regularization/mmd": 0.1944969892501831,
|
|
"regularization/rkhs_norm": 231.54405212402344,
|
|
"regularization/w1": 1.0851236581802368,
|
|
"rewards/chosen": 0.5567921605603449,
|
|
"rewards/margins": 0.8608295813839073,
|
|
"rewards/rejected": -0.3040374208235624,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 17.1892147064209,
|
|
"kl": 14.111940383911133,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33855791.40740741,
|
|
"logits/rejected": -35201098.58573596,
|
|
"logps/chosen": -465.55626006441224,
|
|
"logps/rejected": -373.59080993930195,
|
|
"loss": 0.5627,
|
|
"loss/base_kto": 3.1693296432495117,
|
|
"metrics/advantage_var": 446.8841857910156,
|
|
"regularization/lipschitz_norm": 1349.1328125,
|
|
"regularization/mmd": 0.19930490851402283,
|
|
"regularization/rkhs_norm": 237.26773071289062,
|
|
"regularization/w1": 1.13327157497406,
|
|
"rewards/chosen": 0.5713836129352858,
|
|
"rewards/margins": 0.864196394493518,
|
|
"rewards/rejected": -0.29281278155823215,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 20.75007438659668,
|
|
"kl": 15.750589370727539,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -35293881.32515337,
|
|
"logits/rejected": -35821000.560509555,
|
|
"logps/chosen": -466.9783838190184,
|
|
"logps/rejected": -394.8319566082803,
|
|
"loss": 0.5558,
|
|
"loss/base_kto": 3.197842836380005,
|
|
"metrics/advantage_var": 419.5578308105469,
|
|
"regularization/lipschitz_norm": 1263.475830078125,
|
|
"regularization/mmd": 0.18733568489551544,
|
|
"regularization/rkhs_norm": 223.0186767578125,
|
|
"regularization/w1": 1.0613197088241577,
|
|
"rewards/chosen": 0.48352930735956673,
|
|
"rewards/margins": 0.8397992166776524,
|
|
"rewards/rejected": -0.35626990931808566,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 18.520158767700195,
|
|
"kl": 11.796889305114746,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35220127.79617834,
|
|
"logits/rejected": -36028525.93865031,
|
|
"logps/chosen": -486.74293391719743,
|
|
"logps/rejected": -375.61742714723925,
|
|
"loss": 0.5652,
|
|
"loss/base_kto": 3.2053322792053223,
|
|
"metrics/advantage_var": 438.5023498535156,
|
|
"regularization/lipschitz_norm": 1335.5159912109375,
|
|
"regularization/mmd": 0.19403968751430511,
|
|
"regularization/rkhs_norm": 230.9996337890625,
|
|
"regularization/w1": 1.1218335628509521,
|
|
"rewards/chosen": 0.5484309105356787,
|
|
"rewards/margins": 0.8387103223664638,
|
|
"rewards/rejected": -0.2902794118307851,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 13.852124214172363,
|
|
"kl": 12.641082763671875,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34629694.24924012,
|
|
"logits/rejected": -36090721.95498392,
|
|
"logps/chosen": -488.9081022036474,
|
|
"logps/rejected": -382.36211314308684,
|
|
"loss": 0.5489,
|
|
"loss/base_kto": 3.1825757026672363,
|
|
"metrics/advantage_var": 392.0561828613281,
|
|
"regularization/lipschitz_norm": 1216.16796875,
|
|
"regularization/mmd": 0.18730561435222626,
|
|
"regularization/rkhs_norm": 222.9828643798828,
|
|
"regularization/w1": 1.0215810537338257,
|
|
"rewards/chosen": 0.495199835409147,
|
|
"rewards/margins": 0.8462143682527337,
|
|
"rewards/rejected": -0.3510145328435867,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 13.145265579223633,
|
|
"kl": 12.742012023925781,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -33609220.61261261,
|
|
"logits/rejected": -35441523.90879479,
|
|
"logps/chosen": -485.61425487987987,
|
|
"logps/rejected": -370.5197984527687,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.175414800643921,
|
|
"metrics/advantage_var": 430.4765319824219,
|
|
"regularization/lipschitz_norm": 1363.9232177734375,
|
|
"regularization/mmd": 0.19717474281787872,
|
|
"regularization/rkhs_norm": 234.73184204101562,
|
|
"regularization/w1": 1.1456955671310425,
|
|
"rewards/chosen": 0.5673646210908174,
|
|
"rewards/margins": 0.8735518562737876,
|
|
"rewards/rejected": -0.30618723518297025,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 14.446263313293457,
|
|
"kl": 12.519579887390137,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34501147.44257274,
|
|
"logits/rejected": -35055099.91706539,
|
|
"logps/chosen": -489.0198124042879,
|
|
"logps/rejected": -369.31708532695376,
|
|
"loss": 0.5605,
|
|
"loss/base_kto": 3.2099685668945312,
|
|
"metrics/advantage_var": 428.7497253417969,
|
|
"regularization/lipschitz_norm": 1285.4044189453125,
|
|
"regularization/mmd": 0.19443576037883759,
|
|
"regularization/rkhs_norm": 231.4711151123047,
|
|
"regularization/w1": 1.0797398090362549,
|
|
"rewards/chosen": 0.5657736061177977,
|
|
"rewards/margins": 0.8485941905990203,
|
|
"rewards/rejected": -0.2828205844812226,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 13.536280632019043,
|
|
"kl": 14.868743896484375,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34013258.41390205,
|
|
"logits/rejected": -35506039.88871716,
|
|
"logps/chosen": -453.9443127962085,
|
|
"logps/rejected": -375.7507486476043,
|
|
"loss": 0.5591,
|
|
"loss/base_kto": 3.223512649536133,
|
|
"metrics/advantage_var": 398.3023376464844,
|
|
"regularization/lipschitz_norm": 1263.544189453125,
|
|
"regularization/mmd": 0.18802706897258759,
|
|
"regularization/rkhs_norm": 223.8417510986328,
|
|
"regularization/w1": 1.0613771677017212,
|
|
"rewards/chosen": 0.5673392297343997,
|
|
"rewards/margins": 0.7916319435895122,
|
|
"rewards/rejected": -0.22429271385511254,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 14.502313613891602,
|
|
"kl": 16.450363159179688,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34611026.386185244,
|
|
"logits/rejected": -36537320.11197512,
|
|
"logps/chosen": -482.7138932496075,
|
|
"logps/rejected": -383.52991349144634,
|
|
"loss": 0.5555,
|
|
"loss/base_kto": 3.1732280254364014,
|
|
"metrics/advantage_var": 426.1412048339844,
|
|
"regularization/lipschitz_norm": 1283.1588134765625,
|
|
"regularization/mmd": 0.1930054873228073,
|
|
"regularization/rkhs_norm": 229.76841735839844,
|
|
"regularization/w1": 1.0778534412384033,
|
|
"rewards/chosen": 0.5560097686911303,
|
|
"rewards/margins": 0.8403388167173502,
|
|
"rewards/rejected": -0.2843290480262199,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 21.743240356445312,
|
|
"kl": 16.845134735107422,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -33126860.881141044,
|
|
"logits/rejected": -35962286.74268105,
|
|
"logps/chosen": -492.361529318542,
|
|
"logps/rejected": -365.0382318952234,
|
|
"loss": 0.5529,
|
|
"loss/base_kto": 3.1860368251800537,
|
|
"metrics/advantage_var": 422.7635803222656,
|
|
"regularization/lipschitz_norm": 1245.07568359375,
|
|
"regularization/mmd": 0.1915583610534668,
|
|
"regularization/rkhs_norm": 228.045654296875,
|
|
"regularization/w1": 1.0458635091781616,
|
|
"rewards/chosen": 0.5459863629469592,
|
|
"rewards/margins": 0.8514440438593003,
|
|
"rewards/rejected": -0.3054576809123411,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 17.54193115234375,
|
|
"kl": 15.537714004516602,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34677190.757763974,
|
|
"logits/rejected": -35669922.6163522,
|
|
"logps/chosen": -481.9429833074534,
|
|
"logps/rejected": -383.13536753144655,
|
|
"loss": 0.563,
|
|
"loss/base_kto": 3.186859369277954,
|
|
"metrics/advantage_var": 453.5262756347656,
|
|
"regularization/lipschitz_norm": 1331.4178466796875,
|
|
"regularization/mmd": 0.19879555702209473,
|
|
"regularization/rkhs_norm": 236.661376953125,
|
|
"regularization/w1": 1.1183909177780151,
|
|
"rewards/chosen": 0.5702908438925417,
|
|
"rewards/margins": 0.854516403726956,
|
|
"rewards/rejected": -0.28422555983441433,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 15.706538200378418,
|
|
"kl": 13.580639839172363,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34258531.93993994,
|
|
"logits/rejected": -36151379.38762215,
|
|
"logps/chosen": -488.109375,
|
|
"logps/rejected": -366.2739973534202,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.148057460784912,
|
|
"metrics/advantage_var": 436.72119140625,
|
|
"regularization/lipschitz_norm": 1302.8299560546875,
|
|
"regularization/mmd": 0.19719727337360382,
|
|
"regularization/rkhs_norm": 234.7586669921875,
|
|
"regularization/w1": 1.0943771600723267,
|
|
"rewards/chosen": 0.6121360618430931,
|
|
"rewards/margins": 0.8884072730509769,
|
|
"rewards/rejected": -0.2762712112078837,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 15.945714950561523,
|
|
"kl": 15.48679256439209,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -33477265.341935486,
|
|
"logits/rejected": -35727934.06060606,
|
|
"logps/chosen": -487.28397177419356,
|
|
"logps/rejected": -381.74251893939396,
|
|
"loss": 0.5554,
|
|
"loss/base_kto": 3.197526216506958,
|
|
"metrics/advantage_var": 382.0160827636719,
|
|
"regularization/lipschitz_norm": 1259.7698974609375,
|
|
"regularization/mmd": 0.18740373849868774,
|
|
"regularization/rkhs_norm": 223.09971618652344,
|
|
"regularization/w1": 1.0582067966461182,
|
|
"rewards/chosen": 0.5319727251606603,
|
|
"rewards/margins": 0.8168592645042919,
|
|
"rewards/rejected": -0.2848865393436316,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 14.116644859313965,
|
|
"kl": 12.542868614196777,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34326290.14781297,
|
|
"logits/rejected": -36002339.68233387,
|
|
"logps/chosen": -491.9751602564103,
|
|
"logps/rejected": -387.92184967585086,
|
|
"loss": 0.5657,
|
|
"loss/base_kto": 3.202550172805786,
|
|
"metrics/advantage_var": 420.9132385253906,
|
|
"regularization/lipschitz_norm": 1341.544921875,
|
|
"regularization/mmd": 0.1959245502948761,
|
|
"regularization/rkhs_norm": 233.24351501464844,
|
|
"regularization/w1": 1.1268978118896484,
|
|
"rewards/chosen": 0.5192428289675245,
|
|
"rewards/margins": 0.811094150888108,
|
|
"rewards/rejected": -0.2918513219205835,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 15.54608154296875,
|
|
"kl": 15.73474407196045,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35531926.01273885,
|
|
"logits/rejected": -36403865.91411043,
|
|
"logps/chosen": -491.41983479299364,
|
|
"logps/rejected": -370.36450345092027,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.1753265857696533,
|
|
"metrics/advantage_var": 405.5484313964844,
|
|
"regularization/lipschitz_norm": 1290.739501953125,
|
|
"regularization/mmd": 0.19158188998699188,
|
|
"regularization/rkhs_norm": 228.0736846923828,
|
|
"regularization/w1": 1.0842211246490479,
|
|
"rewards/chosen": 0.5327730604038117,
|
|
"rewards/margins": 0.8599869796396964,
|
|
"rewards/rejected": -0.3272139192358848,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 14.386474609375,
|
|
"kl": 13.991683006286621,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35624960.0,
|
|
"logits/rejected": -36200556.46105919,
|
|
"logps/chosen": -484.3078957680251,
|
|
"logps/rejected": -366.23009151090343,
|
|
"loss": 0.554,
|
|
"loss/base_kto": 3.1901745796203613,
|
|
"metrics/advantage_var": 404.5918884277344,
|
|
"regularization/lipschitz_norm": 1253.319580078125,
|
|
"regularization/mmd": 0.18866340816020966,
|
|
"regularization/rkhs_norm": 224.5992889404297,
|
|
"regularization/w1": 1.0527886152267456,
|
|
"rewards/chosen": 0.5278572871767241,
|
|
"rewards/margins": 0.8375256409615883,
|
|
"rewards/rejected": -0.3096683537848642,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 16.03128433227539,
|
|
"kl": 15.179038047790527,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33715310.479871176,
|
|
"logits/rejected": -34859570.50075872,
|
|
"logps/chosen": -489.4578301127214,
|
|
"logps/rejected": -382.5035091047041,
|
|
"loss": 0.5557,
|
|
"loss/base_kto": 3.1808385848999023,
|
|
"metrics/advantage_var": 406.5344543457031,
|
|
"regularization/lipschitz_norm": 1280.5325927734375,
|
|
"regularization/mmd": 0.18936872482299805,
|
|
"regularization/rkhs_norm": 225.43896484375,
|
|
"regularization/w1": 1.0756473541259766,
|
|
"rewards/chosen": 0.5651349299579811,
|
|
"rewards/margins": 0.8417423449792609,
|
|
"rewards/rejected": -0.2766074150212799,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 17.106462478637695,
|
|
"kl": 15.91718578338623,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34661180.4789644,
|
|
"logits/rejected": -34755144.700906344,
|
|
"logps/chosen": -454.25151699029124,
|
|
"logps/rejected": -389.61961858006043,
|
|
"loss": 0.5563,
|
|
"loss/base_kto": 3.1529488563537598,
|
|
"metrics/advantage_var": 445.4698181152344,
|
|
"regularization/lipschitz_norm": 1311.3211669921875,
|
|
"regularization/mmd": 0.19628138840198517,
|
|
"regularization/rkhs_norm": 233.66831970214844,
|
|
"regularization/w1": 1.101509928703308,
|
|
"rewards/chosen": 0.5545109128489077,
|
|
"rewards/margins": 0.9146029606871937,
|
|
"rewards/rejected": -0.36009204783828597,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 17.375688552856445,
|
|
"kl": 14.020922660827637,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34085769.39641109,
|
|
"logits/rejected": -35753471.23238381,
|
|
"logps/chosen": -469.96818923327896,
|
|
"logps/rejected": -350.6618487631184,
|
|
"loss": 0.5568,
|
|
"loss/base_kto": 3.1888301372528076,
|
|
"metrics/advantage_var": 385.19537353515625,
|
|
"regularization/lipschitz_norm": 1283.8055419921875,
|
|
"regularization/mmd": 0.18733106553554535,
|
|
"regularization/rkhs_norm": 223.01316833496094,
|
|
"regularization/w1": 1.0783965587615967,
|
|
"rewards/chosen": 0.5275428361254842,
|
|
"rewards/margins": 0.8230447967540566,
|
|
"rewards/rejected": -0.2955019606285724,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 16.014177322387695,
|
|
"kl": 11.869200706481934,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -35327283.82822086,
|
|
"logits/rejected": -33905820.53503185,
|
|
"logps/chosen": -443.2274731595092,
|
|
"logps/rejected": -357.488181727707,
|
|
"loss": 0.5615,
|
|
"loss/base_kto": 3.2101943492889404,
|
|
"metrics/advantage_var": 410.871826171875,
|
|
"regularization/lipschitz_norm": 1297.1297607421875,
|
|
"regularization/mmd": 0.19185256958007812,
|
|
"regularization/rkhs_norm": 228.39590454101562,
|
|
"regularization/w1": 1.08958899974823,
|
|
"rewards/chosen": 0.5072993647101467,
|
|
"rewards/margins": 0.8425749539909737,
|
|
"rewards/rejected": -0.335275589280827,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 18.9195613861084,
|
|
"kl": 15.337427139282227,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34568424.21417323,
|
|
"logits/rejected": -35940101.159689926,
|
|
"logps/chosen": -468.53543307086613,
|
|
"logps/rejected": -375.7328003875969,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.189122438430786,
|
|
"metrics/advantage_var": 379.7440185546875,
|
|
"regularization/lipschitz_norm": 1284.2718505859375,
|
|
"regularization/mmd": 0.18473811447620392,
|
|
"regularization/rkhs_norm": 219.9263153076172,
|
|
"regularization/w1": 1.0787882804870605,
|
|
"rewards/chosen": 0.5099421944205217,
|
|
"rewards/margins": 0.8381716638972658,
|
|
"rewards/rejected": -0.3282294694767442,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_loss": 0.5867518629228925,
|
|
"train_runtime": 11108.9609,
|
|
"train_samples_per_second": 13.342,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|