7afb99294c
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed2 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 21.40068244934082,
|
|
"kl": 10.750271797180176,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36277443.12440945,
|
|
"logits/rejected": -37590417.662015505,
|
|
"logps/chosen": -482.0126968503937,
|
|
"logps/rejected": -355.0290455426357,
|
|
"loss": 0.6257,
|
|
"loss/base_kto": 3.9846127033233643,
|
|
"metrics/advantage_var": 306.07135009765625,
|
|
"regularization/lipschitz_norm": 1038.2750244140625,
|
|
"regularization/mmd": 0.14890654385089874,
|
|
"regularization/rkhs_norm": 177.26968383789062,
|
|
"regularization/w1": 0.8721510171890259,
|
|
"rewards/chosen": 0.049396277600386014,
|
|
"rewards/margins": 0.016674185925581328,
|
|
"rewards/rejected": 0.032722091674804686,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 20.915592193603516,
|
|
"kl": 5.919198036193848,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35934931.2,
|
|
"logits/rejected": -36649977.6,
|
|
"logps/chosen": -476.497265625,
|
|
"logps/rejected": -412.370166015625,
|
|
"loss": 0.6069,
|
|
"loss/base_kto": 3.9325339794158936,
|
|
"metrics/advantage_var": 195.82591247558594,
|
|
"regularization/lipschitz_norm": 938.1893920898438,
|
|
"regularization/mmd": 0.1347268670797348,
|
|
"regularization/rkhs_norm": 160.3891143798828,
|
|
"regularization/w1": 0.7880791425704956,
|
|
"rewards/chosen": -0.0020825549960136413,
|
|
"rewards/margins": 0.07140383273363113,
|
|
"rewards/rejected": -0.07348638772964478,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 23.274030685424805,
|
|
"kl": 4.183704853057861,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35996404.72955975,
|
|
"logits/rejected": -38320554.136645965,
|
|
"logps/chosen": -488.87952044025155,
|
|
"logps/rejected": -374.0691721661491,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.9307668209075928,
|
|
"metrics/advantage_var": 153.05758666992188,
|
|
"regularization/lipschitz_norm": 782.2279663085938,
|
|
"regularization/mmd": 0.11708618700504303,
|
|
"regularization/rkhs_norm": 139.38832092285156,
|
|
"regularization/w1": 0.6570715308189392,
|
|
"rewards/chosen": -0.05881994475358687,
|
|
"rewards/margins": 0.06567056807061974,
|
|
"rewards/rejected": -0.12449051282420662,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 22.874296188354492,
|
|
"kl": 3.3950088024139404,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37939921.3081761,
|
|
"logits/rejected": -37839290.03726708,
|
|
"logps/chosen": -505.8852201257862,
|
|
"logps/rejected": -384.4646253881988,
|
|
"loss": 0.5999,
|
|
"loss/base_kto": 3.9200236797332764,
|
|
"metrics/advantage_var": 171.45558166503906,
|
|
"regularization/lipschitz_norm": 895.6095581054688,
|
|
"regularization/mmd": 0.12681341171264648,
|
|
"regularization/rkhs_norm": 150.9683380126953,
|
|
"regularization/w1": 0.7523121237754822,
|
|
"rewards/chosen": -0.018644677767963528,
|
|
"rewards/margins": 0.06818433001458755,
|
|
"rewards/rejected": -0.08682900778255108,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 41.386688232421875,
|
|
"kl": 12.982012748718262,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36996718.05607477,
|
|
"logits/rejected": -37555254.57053292,
|
|
"logps/chosen": -507.32827102803736,
|
|
"logps/rejected": -378.2987852664577,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.8562443256378174,
|
|
"metrics/advantage_var": 200.36048889160156,
|
|
"regularization/lipschitz_norm": 967.3966064453125,
|
|
"regularization/mmd": 0.13394849002361298,
|
|
"regularization/rkhs_norm": 159.4624786376953,
|
|
"regularization/w1": 0.8126131296157837,
|
|
"rewards/chosen": 0.20962636121708283,
|
|
"rewards/margins": 0.13548541190802377,
|
|
"rewards/rejected": 0.07414094930905907,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 25.86820411682129,
|
|
"kl": 6.911703586578369,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36390491.00463679,
|
|
"logits/rejected": -37857732.95418642,
|
|
"logps/chosen": -469.4181317619784,
|
|
"logps/rejected": -370.1704186413902,
|
|
"loss": 0.5958,
|
|
"loss/base_kto": 3.8432776927948,
|
|
"metrics/advantage_var": 205.13658142089844,
|
|
"regularization/lipschitz_norm": 939.0703125,
|
|
"regularization/mmd": 0.1340487003326416,
|
|
"regularization/rkhs_norm": 159.581787109375,
|
|
"regularization/w1": 0.7888190150260925,
|
|
"rewards/chosen": 0.09037629824701748,
|
|
"rewards/margins": 0.13546952727869327,
|
|
"rewards/rejected": -0.0450932290316758,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 18.997695922851562,
|
|
"kl": 12.906949043273926,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36729201.516819574,
|
|
"logits/rejected": -38149044.75399361,
|
|
"logps/chosen": -478.9439029051988,
|
|
"logps/rejected": -383.27840455271564,
|
|
"loss": 0.5994,
|
|
"loss/base_kto": 3.8582570552825928,
|
|
"metrics/advantage_var": 225.24111938476562,
|
|
"regularization/lipschitz_norm": 950.2708129882812,
|
|
"regularization/mmd": 0.139038547873497,
|
|
"regularization/rkhs_norm": 165.5220947265625,
|
|
"regularization/w1": 0.7982274889945984,
|
|
"rewards/chosen": 0.21457646454510704,
|
|
"rewards/margins": 0.10928962563579943,
|
|
"rewards/rejected": 0.10528683890930761,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 19.32913589477539,
|
|
"kl": 7.309741497039795,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36090498.75692308,
|
|
"logits/rejected": -37735586.53968254,
|
|
"logps/chosen": -494.3607692307692,
|
|
"logps/rejected": -360.7294146825397,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.840000867843628,
|
|
"metrics/advantage_var": 220.40237426757812,
|
|
"regularization/lipschitz_norm": 964.1617431640625,
|
|
"regularization/mmd": 0.13911890983581543,
|
|
"regularization/rkhs_norm": 165.6177520751953,
|
|
"regularization/w1": 0.8098956942558289,
|
|
"rewards/chosen": 0.10057786207932692,
|
|
"rewards/margins": 0.14170912084707557,
|
|
"rewards/rejected": -0.04113125876774864,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 23.91461944580078,
|
|
"kl": 20.435789108276367,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35669900.97971919,
|
|
"logits/rejected": -36815447.33646322,
|
|
"logps/chosen": -524.1534711388456,
|
|
"logps/rejected": -364.82325899843505,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.7847213745117188,
|
|
"metrics/advantage_var": 219.2493133544922,
|
|
"regularization/lipschitz_norm": 1033.218505859375,
|
|
"regularization/mmd": 0.1453201025724411,
|
|
"regularization/rkhs_norm": 173.0001220703125,
|
|
"regularization/w1": 0.8679035305976868,
|
|
"rewards/chosen": 0.4183949010792463,
|
|
"rewards/margins": 0.1932723149766935,
|
|
"rewards/rejected": 0.2251225861025528,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 19.60841178894043,
|
|
"kl": 9.85562801361084,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -35645330.343307085,
|
|
"logits/rejected": -37212804.56434108,
|
|
"logps/chosen": -461.6384842519685,
|
|
"logps/rejected": -373.93183139534887,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.8076469898223877,
|
|
"metrics/advantage_var": 234.4076690673828,
|
|
"regularization/lipschitz_norm": 1002.0093994140625,
|
|
"regularization/mmd": 0.1420026272535324,
|
|
"regularization/rkhs_norm": 169.05075073242188,
|
|
"regularization/w1": 0.8416878581047058,
|
|
"rewards/chosen": 0.15029915636918675,
|
|
"rewards/margins": 0.18358521702772296,
|
|
"rewards/rejected": -0.03328606065853622,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 20.37142562866211,
|
|
"kl": 8.391122817993164,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36988588.820189275,
|
|
"logits/rejected": -38302051.071207434,
|
|
"logps/chosen": -476.42522673501577,
|
|
"logps/rejected": -378.99342105263156,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.7933616638183594,
|
|
"metrics/advantage_var": 214.8227996826172,
|
|
"regularization/lipschitz_norm": 978.1011962890625,
|
|
"regularization/mmd": 0.13980741798877716,
|
|
"regularization/rkhs_norm": 166.43740844726562,
|
|
"regularization/w1": 0.8216049075126648,
|
|
"rewards/chosen": 0.14860421550762767,
|
|
"rewards/margins": 0.20199881657957885,
|
|
"rewards/rejected": -0.05339460107195119,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 22.483036041259766,
|
|
"kl": 3.584221601486206,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -34606272.927536234,
|
|
"logits/rejected": -37273354.48861912,
|
|
"logps/chosen": -483.32402375201286,
|
|
"logps/rejected": -351.48380595599394,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.776458740234375,
|
|
"metrics/advantage_var": 235.9140625,
|
|
"regularization/lipschitz_norm": 1024.95703125,
|
|
"regularization/mmd": 0.14718583226203918,
|
|
"regularization/rkhs_norm": 175.22122192382812,
|
|
"regularization/w1": 0.8609639406204224,
|
|
"rewards/chosen": -0.0043065060356005,
|
|
"rewards/margins": 0.2134574381358477,
|
|
"rewards/rejected": -0.2177639441714482,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 21.719362258911133,
|
|
"kl": 5.35298490524292,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37231083.003154576,
|
|
"logits/rejected": -37633971.913312696,
|
|
"logps/chosen": -484.7230875394322,
|
|
"logps/rejected": -390.4203753869969,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.8078582286834717,
|
|
"metrics/advantage_var": 224.3940887451172,
|
|
"regularization/lipschitz_norm": 1006.6205444335938,
|
|
"regularization/mmd": 0.14446866512298584,
|
|
"regularization/rkhs_norm": 171.98651123046875,
|
|
"regularization/w1": 0.8455612063407898,
|
|
"rewards/chosen": 0.015058782198451671,
|
|
"rewards/margins": 0.19098797867723336,
|
|
"rewards/rejected": -0.1759291964787817,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 23.588197708129883,
|
|
"kl": 9.649115562438965,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36851007.41284404,
|
|
"logits/rejected": -38255985.68690096,
|
|
"logps/chosen": -493.18969801223244,
|
|
"logps/rejected": -375.9551467651757,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.789781332015991,
|
|
"metrics/advantage_var": 222.56387329101562,
|
|
"regularization/lipschitz_norm": 986.0309448242188,
|
|
"regularization/mmd": 0.14440391957759857,
|
|
"regularization/rkhs_norm": 171.90943908691406,
|
|
"regularization/w1": 0.8282660841941833,
|
|
"rewards/chosen": 0.20194713347548737,
|
|
"rewards/margins": 0.20969576167135057,
|
|
"rewards/rejected": -0.007748628195863181,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 19.09756088256836,
|
|
"kl": 7.472548961639404,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -34675176.2432,
|
|
"logits/rejected": -36000519.4259542,
|
|
"logps/chosen": -496.2152,
|
|
"logps/rejected": -397.30441316793895,
|
|
"loss": 0.5994,
|
|
"loss/base_kto": 3.7931594848632812,
|
|
"metrics/advantage_var": 237.2469940185547,
|
|
"regularization/lipschitz_norm": 1019.13427734375,
|
|
"regularization/mmd": 0.14561642706394196,
|
|
"regularization/rkhs_norm": 173.35289001464844,
|
|
"regularization/w1": 0.8560728430747986,
|
|
"rewards/chosen": 0.1206897216796875,
|
|
"rewards/margins": 0.18732220943538286,
|
|
"rewards/rejected": -0.06663248775569537,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 21.29581069946289,
|
|
"kl": 4.23238468170166,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35209758.495297804,
|
|
"logits/rejected": -36530865.04672897,
|
|
"logps/chosen": -472.90654388714734,
|
|
"logps/rejected": -366.8334306853583,
|
|
"loss": 0.59,
|
|
"loss/base_kto": 3.759244203567505,
|
|
"metrics/advantage_var": 218.84646606445312,
|
|
"regularization/lipschitz_norm": 975.462890625,
|
|
"regularization/mmd": 0.14115722477436066,
|
|
"regularization/rkhs_norm": 168.0443115234375,
|
|
"regularization/w1": 0.8193888068199158,
|
|
"rewards/chosen": -0.024968752666700598,
|
|
"rewards/margins": 0.23026391867286325,
|
|
"rewards/rejected": -0.25523267133956384,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 22.018970489501953,
|
|
"kl": 7.800880432128906,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36179909.835575484,
|
|
"logits/rejected": -37338847.738134205,
|
|
"logps/chosen": -456.9335762331838,
|
|
"logps/rejected": -360.0439341243862,
|
|
"loss": 0.5907,
|
|
"loss/base_kto": 3.7180793285369873,
|
|
"metrics/advantage_var": 241.9764862060547,
|
|
"regularization/lipschitz_norm": 1023.1056518554688,
|
|
"regularization/mmd": 0.14846424758434296,
|
|
"regularization/rkhs_norm": 176.74314880371094,
|
|
"regularization/w1": 0.859408974647522,
|
|
"rewards/chosen": 0.14203581075853652,
|
|
"rewards/margins": 0.276979271825499,
|
|
"rewards/rejected": -0.13494346106696245,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 21.33985137939453,
|
|
"kl": 12.41759204864502,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35627460.92307692,
|
|
"logits/rejected": -36198237.65853658,
|
|
"logps/chosen": -484.9215745192308,
|
|
"logps/rejected": -374.1507955411585,
|
|
"loss": 0.5991,
|
|
"loss/base_kto": 3.773512363433838,
|
|
"metrics/advantage_var": 234.97900390625,
|
|
"regularization/lipschitz_norm": 1037.84765625,
|
|
"regularization/mmd": 0.14775054156780243,
|
|
"regularization/rkhs_norm": 175.8935089111328,
|
|
"regularization/w1": 0.8717920184135437,
|
|
"rewards/chosen": 0.14012701083452272,
|
|
"rewards/margins": 0.18075988969629894,
|
|
"rewards/rejected": -0.04063287886177621,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 17.242639541625977,
|
|
"kl": 13.246150016784668,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -37563619.381317,
|
|
"logits/rejected": -37545647.5661882,
|
|
"logps/chosen": -482.21817572741196,
|
|
"logps/rejected": -371.72552830940987,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.7767891883850098,
|
|
"metrics/advantage_var": 239.0876922607422,
|
|
"regularization/lipschitz_norm": 1003.5947265625,
|
|
"regularization/mmd": 0.14514389634132385,
|
|
"regularization/rkhs_norm": 172.7903594970703,
|
|
"regularization/w1": 0.8430196642875671,
|
|
"rewards/chosen": 0.21783804783959848,
|
|
"rewards/margins": 0.19418382344993057,
|
|
"rewards/rejected": 0.023654224389667907,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 16.8441162109375,
|
|
"kl": 6.955906867980957,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36842864.236453205,
|
|
"logits/rejected": -38858245.34128167,
|
|
"logps/chosen": -492.00656814449917,
|
|
"logps/rejected": -375.56701751117737,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.7773163318634033,
|
|
"metrics/advantage_var": 224.1083526611328,
|
|
"regularization/lipschitz_norm": 966.634765625,
|
|
"regularization/mmd": 0.14453430473804474,
|
|
"regularization/rkhs_norm": 172.0646514892578,
|
|
"regularization/w1": 0.8119732141494751,
|
|
"rewards/chosen": 0.08100843507863814,
|
|
"rewards/margins": 0.2072963587408494,
|
|
"rewards/rejected": -0.12628792366221125,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 19.023828506469727,
|
|
"kl": 7.648221015930176,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36642315.79723502,
|
|
"logits/rejected": -37101613.58346582,
|
|
"logps/chosen": -490.3146121351767,
|
|
"logps/rejected": -367.69815679650236,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.722442626953125,
|
|
"metrics/advantage_var": 259.7574157714844,
|
|
"regularization/lipschitz_norm": 1068.7039794921875,
|
|
"regularization/mmd": 0.15702545642852783,
|
|
"regularization/rkhs_norm": 186.93508911132812,
|
|
"regularization/w1": 0.8977112770080566,
|
|
"rewards/chosen": 0.1375455607283866,
|
|
"rewards/margins": 0.279032536757581,
|
|
"rewards/rejected": -0.1414869760291944,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 19.267927169799805,
|
|
"kl": 8.824287414550781,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35695188.56456456,
|
|
"logits/rejected": -38848785.511400655,
|
|
"logps/chosen": -508.701341966967,
|
|
"logps/rejected": -362.9134771986971,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.7587311267852783,
|
|
"metrics/advantage_var": 268.5464782714844,
|
|
"regularization/lipschitz_norm": 1049.3856201171875,
|
|
"regularization/mmd": 0.15315692126750946,
|
|
"regularization/rkhs_norm": 182.3296661376953,
|
|
"regularization/w1": 0.8814839720726013,
|
|
"rewards/chosen": 0.055987710351342555,
|
|
"rewards/margins": 0.2602794112798722,
|
|
"rewards/rejected": -0.20429170092852963,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 24.938785552978516,
|
|
"kl": 14.582898139953613,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35797747.220611915,
|
|
"logits/rejected": -37555540.29742033,
|
|
"logps/chosen": -472.09450483091786,
|
|
"logps/rejected": -352.59623956752654,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.766038656234741,
|
|
"metrics/advantage_var": 246.08164978027344,
|
|
"regularization/lipschitz_norm": 1015.1571655273438,
|
|
"regularization/mmd": 0.14946337044239044,
|
|
"regularization/rkhs_norm": 177.93260192871094,
|
|
"regularization/w1": 0.8527318835258484,
|
|
"rewards/chosen": 0.27431054353330064,
|
|
"rewards/margins": 0.22588981836062394,
|
|
"rewards/rejected": 0.0484207251726767,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.859481811523438,
|
|
"kl": 6.946757793426514,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37414401.64630225,
|
|
"logits/rejected": -38209224.753799394,
|
|
"logps/chosen": -484.1299236334405,
|
|
"logps/rejected": -387.6064542173252,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.733232259750366,
|
|
"metrics/advantage_var": 254.38967895507812,
|
|
"regularization/lipschitz_norm": 1019.4359741210938,
|
|
"regularization/mmd": 0.15039943158626556,
|
|
"regularization/rkhs_norm": 179.04693603515625,
|
|
"regularization/w1": 0.8563261032104492,
|
|
"rewards/chosen": 0.0983930078733389,
|
|
"rewards/margins": 0.25236842795846903,
|
|
"rewards/rejected": -0.15397542008513013,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.229341506958008,
|
|
"kl": 5.760401725769043,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36467886.36532508,
|
|
"logits/rejected": -37971590.056782335,
|
|
"logps/chosen": -499.09911958204333,
|
|
"logps/rejected": -356.59375,
|
|
"loss": 0.5993,
|
|
"loss/base_kto": 3.7415833473205566,
|
|
"metrics/advantage_var": 274.0146484375,
|
|
"regularization/lipschitz_norm": 1066.2662353515625,
|
|
"regularization/mmd": 0.1567559540271759,
|
|
"regularization/rkhs_norm": 186.61424255371094,
|
|
"regularization/w1": 0.8956636786460876,
|
|
"rewards/chosen": 0.0302486449203255,
|
|
"rewards/margins": 0.24224814701207925,
|
|
"rewards/rejected": -0.21199950209175375,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 17.295522689819336,
|
|
"kl": 11.863730430603027,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36054879.06317411,
|
|
"logits/rejected": -36389237.24881141,
|
|
"logps/chosen": -485.5292758089368,
|
|
"logps/rejected": -367.02431656101425,
|
|
"loss": 0.5913,
|
|
"loss/base_kto": 3.703291416168213,
|
|
"metrics/advantage_var": 243.568359375,
|
|
"regularization/lipschitz_norm": 1047.3997802734375,
|
|
"regularization/mmd": 0.14756233990192413,
|
|
"regularization/rkhs_norm": 175.66944885253906,
|
|
"regularization/w1": 0.8798158764839172,
|
|
"rewards/chosen": 0.19453550083060478,
|
|
"rewards/margins": 0.26168162073576523,
|
|
"rewards/rejected": -0.06714611990516046,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 23.003244400024414,
|
|
"kl": 16.922313690185547,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36577970.89156626,
|
|
"logits/rejected": -36688776.31168831,
|
|
"logps/chosen": -480.8135353915663,
|
|
"logps/rejected": -357.5077617694805,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.7339158058166504,
|
|
"metrics/advantage_var": 220.2282257080078,
|
|
"regularization/lipschitz_norm": 979.9989624023438,
|
|
"regularization/mmd": 0.1428050845861435,
|
|
"regularization/rkhs_norm": 170.0060577392578,
|
|
"regularization/w1": 0.8231990933418274,
|
|
"rewards/chosen": 0.2611128565776779,
|
|
"rewards/margins": 0.2345795315805315,
|
|
"rewards/rejected": 0.026533324997146408,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.125394821166992,
|
|
"kl": 17.437366485595703,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35410925.26829268,
|
|
"logits/rejected": -37331019.48717949,
|
|
"logps/chosen": -466.1077553353659,
|
|
"logps/rejected": -347.48998397435895,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.7208313941955566,
|
|
"metrics/advantage_var": 243.2664337158203,
|
|
"regularization/lipschitz_norm": 1032.2681884765625,
|
|
"regularization/mmd": 0.15240328013896942,
|
|
"regularization/rkhs_norm": 181.43247985839844,
|
|
"regularization/w1": 0.8671053051948547,
|
|
"rewards/chosen": 0.30058223445241045,
|
|
"rewards/margins": 0.25340126088591497,
|
|
"rewards/rejected": 0.047180973566495456,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 19.677757263183594,
|
|
"kl": 21.856731414794922,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37115605.83529412,
|
|
"logits/rejected": -38863783.25333333,
|
|
"logps/chosen": -472.9079044117647,
|
|
"logps/rejected": -384.5609375,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.6904587745666504,
|
|
"metrics/advantage_var": 288.47802734375,
|
|
"regularization/lipschitz_norm": 1129.0067138671875,
|
|
"regularization/mmd": 0.1618659943342209,
|
|
"regularization/rkhs_norm": 192.69761657714844,
|
|
"regularization/w1": 0.9483656287193298,
|
|
"rewards/chosen": 0.41798903521369485,
|
|
"rewards/margins": 0.2497961971806545,
|
|
"rewards/rejected": 0.16819283803304036,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 26.598800659179688,
|
|
"kl": 23.661420822143555,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36961220.13538461,
|
|
"logits/rejected": -36617537.82857143,
|
|
"logps/chosen": -448.7200480769231,
|
|
"logps/rejected": -392.6844742063492,
|
|
"loss": 0.5893,
|
|
"loss/base_kto": 3.745983839035034,
|
|
"metrics/advantage_var": 216.1056365966797,
|
|
"regularization/lipschitz_norm": 981.8429565429688,
|
|
"regularization/mmd": 0.14375527203083038,
|
|
"regularization/rkhs_norm": 171.13720703125,
|
|
"regularization/w1": 0.8247480392456055,
|
|
"rewards/chosen": 0.4046255727914663,
|
|
"rewards/margins": 0.21790316552615135,
|
|
"rewards/rejected": 0.18672240726531497,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 19.250160217285156,
|
|
"kl": 19.82950210571289,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35359013.03144654,
|
|
"logits/rejected": -36822594.782608695,
|
|
"logps/chosen": -485.5686419025157,
|
|
"logps/rejected": -387.8875679347826,
|
|
"loss": 0.5897,
|
|
"loss/base_kto": 3.7229554653167725,
|
|
"metrics/advantage_var": 238.7768096923828,
|
|
"regularization/lipschitz_norm": 1010.4977416992188,
|
|
"regularization/mmd": 0.14569826424121857,
|
|
"regularization/rkhs_norm": 173.45030212402344,
|
|
"regularization/w1": 0.8488180041313171,
|
|
"rewards/chosen": 0.29132831021674777,
|
|
"rewards/margins": 0.23126568260209265,
|
|
"rewards/rejected": 0.06006262761465511,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 15.11947250366211,
|
|
"kl": 17.816537857055664,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -34935488.72374798,
|
|
"logits/rejected": -36300338.34795764,
|
|
"logps/chosen": -492.0254442649435,
|
|
"logps/rejected": -366.63788294251134,
|
|
"loss": 0.5872,
|
|
"loss/base_kto": 3.688230514526367,
|
|
"metrics/advantage_var": 235.89317321777344,
|
|
"regularization/lipschitz_norm": 1025.1229248046875,
|
|
"regularization/mmd": 0.1482767015695572,
|
|
"regularization/rkhs_norm": 176.51988220214844,
|
|
"regularization/w1": 0.8611032366752625,
|
|
"rewards/chosen": 0.26340988726146003,
|
|
"rewards/margins": 0.2826291342686136,
|
|
"rewards/rejected": -0.019219247007153578,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 22.04835319519043,
|
|
"kl": 17.506006240844727,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36742247.33944954,
|
|
"logits/rejected": -37994522.17252396,
|
|
"logps/chosen": -494.6242354740061,
|
|
"logps/rejected": -370.19131888977637,
|
|
"loss": 0.5914,
|
|
"loss/base_kto": 3.67919921875,
|
|
"metrics/advantage_var": 263.4634094238281,
|
|
"regularization/lipschitz_norm": 1064.8973388671875,
|
|
"regularization/mmd": 0.15754012763500214,
|
|
"regularization/rkhs_norm": 187.5477752685547,
|
|
"regularization/w1": 0.894513726234436,
|
|
"rewards/chosen": 0.28376102739153286,
|
|
"rewards/margins": 0.28764767298587685,
|
|
"rewards/rejected": -0.0038866455943439714,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 20.036449432373047,
|
|
"kl": 16.565162658691406,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35270000.64,
|
|
"logits/rejected": -37403255.59694657,
|
|
"logps/chosen": -473.36035,
|
|
"logps/rejected": -375.27058683206104,
|
|
"loss": 0.589,
|
|
"loss/base_kto": 3.7290191650390625,
|
|
"metrics/advantage_var": 235.22976684570312,
|
|
"regularization/lipschitz_norm": 994.79736328125,
|
|
"regularization/mmd": 0.14701871573925018,
|
|
"regularization/rkhs_norm": 175.0222930908203,
|
|
"regularization/w1": 0.8356297612190247,
|
|
"rewards/chosen": 0.1914079345703125,
|
|
"rewards/margins": 0.22244986758632512,
|
|
"rewards/rejected": -0.031041933016012643,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 17.892250061035156,
|
|
"kl": 8.530532836914062,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -34931034.785942495,
|
|
"logits/rejected": -37552472.46483181,
|
|
"logps/chosen": -484.74790335463257,
|
|
"logps/rejected": -362.5554998088685,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.739649534225464,
|
|
"metrics/advantage_var": 237.2588653564453,
|
|
"regularization/lipschitz_norm": 1031.495849609375,
|
|
"regularization/mmd": 0.1502029150724411,
|
|
"regularization/rkhs_norm": 178.81298828125,
|
|
"regularization/w1": 0.8664564490318298,
|
|
"rewards/chosen": 0.13256341123733276,
|
|
"rewards/margins": 0.23601116495693092,
|
|
"rewards/rejected": -0.10344775371959815,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 19.33931541442871,
|
|
"kl": 4.368712902069092,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36260002.282091916,
|
|
"logits/rejected": -37281905.60246533,
|
|
"logps/chosen": -484.9353209191759,
|
|
"logps/rejected": -371.9441929892142,
|
|
"loss": 0.6012,
|
|
"loss/base_kto": 3.759192705154419,
|
|
"metrics/advantage_var": 262.6382751464844,
|
|
"regularization/lipschitz_norm": 1064.716796875,
|
|
"regularization/mmd": 0.15617389976978302,
|
|
"regularization/rkhs_norm": 185.9213104248047,
|
|
"regularization/w1": 0.8943620920181274,
|
|
"rewards/chosen": -0.04001742875330422,
|
|
"rewards/margins": 0.22285017404908408,
|
|
"rewards/rejected": -0.2628676028023883,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 21.3850154876709,
|
|
"kl": 16.32546615600586,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -37349324.47798742,
|
|
"logits/rejected": -38612114.28571428,
|
|
"logps/chosen": -515.1336477987421,
|
|
"logps/rejected": -352.93759704968943,
|
|
"loss": 0.5952,
|
|
"loss/base_kto": 3.700870990753174,
|
|
"metrics/advantage_var": 261.3362731933594,
|
|
"regularization/lipschitz_norm": 1078.4302978515625,
|
|
"regularization/mmd": 0.15469644963741302,
|
|
"regularization/rkhs_norm": 184.1624298095703,
|
|
"regularization/w1": 0.9058815240859985,
|
|
"rewards/chosen": 0.2217560774125393,
|
|
"rewards/margins": 0.2460698382529138,
|
|
"rewards/rejected": -0.02431376084037449,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 20.18855857849121,
|
|
"kl": 10.308929443359375,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -37575057.76076555,
|
|
"logits/rejected": -36973647.9754977,
|
|
"logps/chosen": -490.06873006379584,
|
|
"logps/rejected": -397.8548047473201,
|
|
"loss": 0.5913,
|
|
"loss/base_kto": 3.7251031398773193,
|
|
"metrics/advantage_var": 254.86929321289062,
|
|
"regularization/lipschitz_norm": 1019.8321533203125,
|
|
"regularization/mmd": 0.1485794186592102,
|
|
"regularization/rkhs_norm": 176.88026428222656,
|
|
"regularization/w1": 0.8566590547561646,
|
|
"rewards/chosen": 0.0450934937886264,
|
|
"rewards/margins": 0.22569197620975365,
|
|
"rewards/rejected": -0.18059848242112725,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 18.445985794067383,
|
|
"kl": 8.989788055419922,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36942340.72615384,
|
|
"logits/rejected": -37296851.97452229,
|
|
"logps/chosen": -480.4367307692308,
|
|
"logps/rejected": -398.917869227707,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.5884010791778564,
|
|
"metrics/advantage_var": 360.9624938964844,
|
|
"regularization/lipschitz_norm": 1214.3770751953125,
|
|
"regularization/mmd": 0.17500752210617065,
|
|
"regularization/rkhs_norm": 208.34231567382812,
|
|
"regularization/w1": 1.0200766324996948,
|
|
"rewards/chosen": 0.20940889798677884,
|
|
"rewards/margins": 0.40099498545054646,
|
|
"rewards/rejected": -0.19158608746376765,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 20.656959533691406,
|
|
"kl": 17.82853126525879,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -37024200.52012384,
|
|
"logits/rejected": -36696529.16088328,
|
|
"logps/chosen": -465.2453560371517,
|
|
"logps/rejected": -383.90164136435334,
|
|
"loss": 0.6084,
|
|
"loss/base_kto": 3.2800545692443848,
|
|
"metrics/advantage_var": 649.1947021484375,
|
|
"regularization/lipschitz_norm": 1615.78662109375,
|
|
"regularization/mmd": 0.2295728176832199,
|
|
"regularization/rkhs_norm": 273.3009948730469,
|
|
"regularization/w1": 1.357260823249817,
|
|
"rewards/chosen": 0.5885597913995985,
|
|
"rewards/margins": 0.8095202460094664,
|
|
"rewards/rejected": -0.2209604546098679,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 19.948137283325195,
|
|
"kl": 7.478545665740967,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35273180.74754098,
|
|
"logits/rejected": -37067244.13134328,
|
|
"logps/chosen": -491.67838114754096,
|
|
"logps/rejected": -370.31473880597014,
|
|
"loss": 0.6285,
|
|
"loss/base_kto": 3.336402177810669,
|
|
"metrics/advantage_var": 705.7559204101562,
|
|
"regularization/lipschitz_norm": 1723.125,
|
|
"regularization/mmd": 0.24384193122386932,
|
|
"regularization/rkhs_norm": 290.2879943847656,
|
|
"regularization/w1": 1.4474248886108398,
|
|
"rewards/chosen": 0.29649758260758197,
|
|
"rewards/margins": 0.7547523597472741,
|
|
"rewards/rejected": -0.45825477713969215,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 18.255462646484375,
|
|
"kl": 11.052966117858887,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36308858.079744816,
|
|
"logits/rejected": -37253345.813169986,
|
|
"logps/chosen": -470.62161084529504,
|
|
"logps/rejected": -355.5348870597243,
|
|
"loss": 0.6091,
|
|
"loss/base_kto": 3.3348100185394287,
|
|
"metrics/advantage_var": 586.0612182617188,
|
|
"regularization/lipschitz_norm": 1566.06640625,
|
|
"regularization/mmd": 0.2221541851758957,
|
|
"regularization/rkhs_norm": 264.46923828125,
|
|
"regularization/w1": 1.3154957294464111,
|
|
"rewards/chosen": 0.28558369078324364,
|
|
"rewards/margins": 0.6870733224711265,
|
|
"rewards/rejected": -0.4014896316878829,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 17.90972137451172,
|
|
"kl": 6.862426280975342,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34103140.7483871,
|
|
"logits/rejected": -35375122.61818182,
|
|
"logps/chosen": -496.13114919354837,
|
|
"logps/rejected": -369.8449337121212,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.3216614723205566,
|
|
"metrics/advantage_var": 492.7622985839844,
|
|
"regularization/lipschitz_norm": 1510.7691650390625,
|
|
"regularization/mmd": 0.20917610824108124,
|
|
"regularization/rkhs_norm": 249.01919555664062,
|
|
"regularization/w1": 1.2690460681915283,
|
|
"rewards/chosen": 0.23620691607075353,
|
|
"rewards/margins": 0.7167317999190016,
|
|
"rewards/rejected": -0.4805248838482481,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 22.013103485107422,
|
|
"kl": 8.058979034423828,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34856904.314199395,
|
|
"logits/rejected": -36060236.22006472,
|
|
"logps/chosen": -492.1848092900302,
|
|
"logps/rejected": -386.75725627022655,
|
|
"loss": 0.6113,
|
|
"loss/base_kto": 3.2472546100616455,
|
|
"metrics/advantage_var": 640.4850463867188,
|
|
"regularization/lipschitz_norm": 1675.7041015625,
|
|
"regularization/mmd": 0.23541966080665588,
|
|
"regularization/rkhs_norm": 280.2615051269531,
|
|
"regularization/w1": 1.407591462135315,
|
|
"rewards/chosen": 0.4598341650833176,
|
|
"rewards/margins": 0.8801655583282862,
|
|
"rewards/rejected": -0.42033139324496865,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.79622459411621,
|
|
"kl": 17.36254119873047,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35095817.481481485,
|
|
"logits/rejected": -36155171.64556962,
|
|
"logps/chosen": -472.74768518518516,
|
|
"logps/rejected": -394.8189527294304,
|
|
"loss": 0.5958,
|
|
"loss/base_kto": 3.2487809658050537,
|
|
"metrics/advantage_var": 673.8466186523438,
|
|
"regularization/lipschitz_norm": 1536.3482666015625,
|
|
"regularization/mmd": 0.22722311317920685,
|
|
"regularization/rkhs_norm": 270.50372314453125,
|
|
"regularization/w1": 1.2905325889587402,
|
|
"rewards/chosen": 0.5056878549081308,
|
|
"rewards/margins": 0.8401391681683662,
|
|
"rewards/rejected": -0.33445131326023536,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 20.47112274169922,
|
|
"kl": 11.17072582244873,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34423304.02507837,
|
|
"logits/rejected": -35855200.49844237,
|
|
"logps/chosen": -473.4647335423198,
|
|
"logps/rejected": -406.92810552959503,
|
|
"loss": 0.6059,
|
|
"loss/base_kto": 3.3349392414093018,
|
|
"metrics/advantage_var": 568.5075073242188,
|
|
"regularization/lipschitz_norm": 1540.0595703125,
|
|
"regularization/mmd": 0.21888212859630585,
|
|
"regularization/rkhs_norm": 260.573974609375,
|
|
"regularization/w1": 1.2936500310897827,
|
|
"rewards/chosen": 0.4613130384104379,
|
|
"rewards/margins": 0.7457824527619518,
|
|
"rewards/rejected": -0.28446941435151385,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.54960823059082,
|
|
"kl": 12.041743278503418,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34648457.6,
|
|
"logits/rejected": -36323424.0,
|
|
"logps/chosen": -470.692431640625,
|
|
"logps/rejected": -385.93017578125,
|
|
"loss": 0.6175,
|
|
"loss/base_kto": 3.2486565113067627,
|
|
"metrics/advantage_var": 767.1946411132812,
|
|
"regularization/lipschitz_norm": 1727.6031494140625,
|
|
"regularization/mmd": 0.2397797852754593,
|
|
"regularization/rkhs_norm": 285.4521179199219,
|
|
"regularization/w1": 1.4511866569519043,
|
|
"rewards/chosen": 0.5065313339233398,
|
|
"rewards/margins": 0.8657758712768555,
|
|
"rewards/rejected": -0.35924453735351564,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 18.49418830871582,
|
|
"kl": 12.273981094360352,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35320511.80185758,
|
|
"logits/rejected": -36050879.39432177,
|
|
"logps/chosen": -478.82077205882354,
|
|
"logps/rejected": -368.18947160883283,
|
|
"loss": 0.6096,
|
|
"loss/base_kto": 3.2334420680999756,
|
|
"metrics/advantage_var": 634.3271484375,
|
|
"regularization/lipschitz_norm": 1677.7933349609375,
|
|
"regularization/mmd": 0.23385129868984222,
|
|
"regularization/rkhs_norm": 278.3943786621094,
|
|
"regularization/w1": 1.409346580505371,
|
|
"rewards/chosen": 0.5548504808738873,
|
|
"rewards/margins": 0.8426393533369813,
|
|
"rewards/rejected": -0.2877888724630939,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 14.608499526977539,
|
|
"kl": 11.084086418151855,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -36303766.06896552,
|
|
"logits/rejected": -35893713.74454828,
|
|
"logps/chosen": -473.60452586206895,
|
|
"logps/rejected": -373.63816686137073,
|
|
"loss": 0.6002,
|
|
"loss/base_kto": 3.2938830852508545,
|
|
"metrics/advantage_var": 579.5693969726562,
|
|
"regularization/lipschitz_norm": 1533.9541015625,
|
|
"regularization/mmd": 0.21941792964935303,
|
|
"regularization/rkhs_norm": 261.2118225097656,
|
|
"regularization/w1": 1.288521647453308,
|
|
"rewards/chosen": 0.4103406947981975,
|
|
"rewards/margins": 0.7620927175146841,
|
|
"rewards/rejected": -0.3517520227164866,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 20.328784942626953,
|
|
"kl": 12.64668083190918,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34867118.58103976,
|
|
"logits/rejected": -34741372.319488816,
|
|
"logps/chosen": -482.0506020642202,
|
|
"logps/rejected": -364.62554912140575,
|
|
"loss": 0.6052,
|
|
"loss/base_kto": 3.243938446044922,
|
|
"metrics/advantage_var": 614.6873779296875,
|
|
"regularization/lipschitz_norm": 1634.752685546875,
|
|
"regularization/mmd": 0.22438573837280273,
|
|
"regularization/rkhs_norm": 267.1258850097656,
|
|
"regularization/w1": 1.3731921911239624,
|
|
"rewards/chosen": 0.46585050343738055,
|
|
"rewards/margins": 0.8295787189566123,
|
|
"rewards/rejected": -0.36372821551923173,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 25.50452995300293,
|
|
"kl": 11.645515441894531,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35676945.39805825,
|
|
"logits/rejected": -37748179.141993955,
|
|
"logps/chosen": -487.37328074433657,
|
|
"logps/rejected": -364.22415502265864,
|
|
"loss": 0.6112,
|
|
"loss/base_kto": 3.2887825965881348,
|
|
"metrics/advantage_var": 618.5621337890625,
|
|
"regularization/lipschitz_norm": 1633.3822021484375,
|
|
"regularization/mmd": 0.2286948263645172,
|
|
"regularization/rkhs_norm": 272.2557067871094,
|
|
"regularization/w1": 1.3720409870147705,
|
|
"rewards/chosen": 0.42719488545143103,
|
|
"rewards/margins": 0.7894209279540274,
|
|
"rewards/rejected": -0.3622260425025963,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 16.46270751953125,
|
|
"kl": 8.545905113220215,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35287136.30094044,
|
|
"logits/rejected": -36820341.23364486,
|
|
"logps/chosen": -482.0362460815047,
|
|
"logps/rejected": -397.1982087227414,
|
|
"loss": 0.6194,
|
|
"loss/base_kto": 3.3153412342071533,
|
|
"metrics/advantage_var": 775.555908203125,
|
|
"regularization/lipschitz_norm": 1666.1802978515625,
|
|
"regularization/mmd": 0.24035584926605225,
|
|
"regularization/rkhs_norm": 286.137939453125,
|
|
"regularization/w1": 1.399591326713562,
|
|
"rewards/chosen": 0.29254174307222275,
|
|
"rewards/margins": 0.743224338583686,
|
|
"rewards/rejected": -0.4506825955114632,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 15.143294334411621,
|
|
"kl": 10.15416431427002,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -36295893.26550079,
|
|
"logits/rejected": -37798979.637480795,
|
|
"logps/chosen": -475.4776430842607,
|
|
"logps/rejected": -374.76483294930875,
|
|
"loss": 0.6056,
|
|
"loss/base_kto": 3.2693042755126953,
|
|
"metrics/advantage_var": 595.8081665039062,
|
|
"regularization/lipschitz_norm": 1602.6072998046875,
|
|
"regularization/mmd": 0.22929513454437256,
|
|
"regularization/rkhs_norm": 272.9704284667969,
|
|
"regularization/w1": 1.346190094947815,
|
|
"rewards/chosen": 0.41563485877881556,
|
|
"rewards/margins": 0.7959627310126385,
|
|
"rewards/rejected": -0.380327872233823,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 17.122385025024414,
|
|
"kl": 12.640464782714844,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35502933.333333336,
|
|
"logits/rejected": -36398540.151898734,
|
|
"logps/chosen": -493.1551890432099,
|
|
"logps/rejected": -367.2513844936709,
|
|
"loss": 0.6073,
|
|
"loss/base_kto": 3.297977924346924,
|
|
"metrics/advantage_var": 622.8240356445312,
|
|
"regularization/lipschitz_norm": 1586.560791015625,
|
|
"regularization/mmd": 0.22801847755908966,
|
|
"regularization/rkhs_norm": 271.4505920410156,
|
|
"regularization/w1": 1.332711100578308,
|
|
"rewards/chosen": 0.4176573105800299,
|
|
"rewards/margins": 0.7669607854891725,
|
|
"rewards/rejected": -0.3493034749091426,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 26.15509033203125,
|
|
"kl": 19.166763305664062,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35286762.409638554,
|
|
"logits/rejected": -36649525.1948052,
|
|
"logps/chosen": -475.7409638554217,
|
|
"logps/rejected": -361.3068181818182,
|
|
"loss": 0.6006,
|
|
"loss/base_kto": 3.2766664028167725,
|
|
"metrics/advantage_var": 596.4932250976562,
|
|
"regularization/lipschitz_norm": 1548.7574462890625,
|
|
"regularization/mmd": 0.22698640823364258,
|
|
"regularization/rkhs_norm": 270.2218933105469,
|
|
"regularization/w1": 1.300956130027771,
|
|
"rewards/chosen": 0.591407132435994,
|
|
"rewards/margins": 0.7337774802888247,
|
|
"rewards/rejected": -0.14237034785283076,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 17.70459747314453,
|
|
"kl": 15.791522026062012,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34167187.39393939,
|
|
"logits/rejected": -35733748.438709676,
|
|
"logps/chosen": -473.5657196969697,
|
|
"logps/rejected": -385.8681199596774,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.3037688732147217,
|
|
"metrics/advantage_var": 530.39599609375,
|
|
"regularization/lipschitz_norm": 1513.2613525390625,
|
|
"regularization/mmd": 0.21634235978126526,
|
|
"regularization/rkhs_norm": 257.5504150390625,
|
|
"regularization/w1": 1.2711395025253296,
|
|
"rewards/chosen": 0.5228803692441998,
|
|
"rewards/margins": 0.7562547864569014,
|
|
"rewards/rejected": -0.2333744172127016,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 19.37577247619629,
|
|
"kl": 12.536513328552246,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34044914.82958199,
|
|
"logits/rejected": -35189187.306990884,
|
|
"logps/chosen": -492.9626205787781,
|
|
"logps/rejected": -358.44433890577506,
|
|
"loss": 0.6009,
|
|
"loss/base_kto": 3.2793610095977783,
|
|
"metrics/advantage_var": 580.1519775390625,
|
|
"regularization/lipschitz_norm": 1552.6600341796875,
|
|
"regularization/mmd": 0.2239704132080078,
|
|
"regularization/rkhs_norm": 266.6314697265625,
|
|
"regularization/w1": 1.3042343854904175,
|
|
"rewards/chosen": 0.4799185504484023,
|
|
"rewards/margins": 0.793345000116846,
|
|
"rewards/rejected": -0.3134264496684437,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.8507080078125,
|
|
"kl": 7.36299467086792,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34269775.208931416,
|
|
"logits/rejected": -36514864.612557426,
|
|
"logps/chosen": -493.01674641148327,
|
|
"logps/rejected": -385.3358537519142,
|
|
"loss": 0.608,
|
|
"loss/base_kto": 3.2961883544921875,
|
|
"metrics/advantage_var": 590.896484375,
|
|
"regularization/lipschitz_norm": 1597.564453125,
|
|
"regularization/mmd": 0.2254566252231598,
|
|
"regularization/rkhs_norm": 268.4007263183594,
|
|
"regularization/w1": 1.3419541120529175,
|
|
"rewards/chosen": 0.3667038182892868,
|
|
"rewards/margins": 0.7921911555039451,
|
|
"rewards/rejected": -0.4254873372146583,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 20.057065963745117,
|
|
"kl": 9.643331527709961,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34130037.915151514,
|
|
"logits/rejected": -36673839.896774195,
|
|
"logps/chosen": -497.07045454545454,
|
|
"logps/rejected": -379.64758064516127,
|
|
"loss": 0.6052,
|
|
"loss/base_kto": 3.2919857501983643,
|
|
"metrics/advantage_var": 593.4031372070312,
|
|
"regularization/lipschitz_norm": 1577.991943359375,
|
|
"regularization/mmd": 0.22429490089416504,
|
|
"regularization/rkhs_norm": 267.0177307128906,
|
|
"regularization/w1": 1.325513243675232,
|
|
"rewards/chosen": 0.43017846309777463,
|
|
"rewards/margins": 0.7811854201211491,
|
|
"rewards/rejected": -0.3510069570233745,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 17.019927978515625,
|
|
"kl": 15.356094360351562,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33818884.115755625,
|
|
"logits/rejected": -36837133.22796353,
|
|
"logps/chosen": -483.9380526527331,
|
|
"logps/rejected": -359.3028827887538,
|
|
"loss": 0.6099,
|
|
"loss/base_kto": 3.34063982963562,
|
|
"metrics/advantage_var": 590.6072387695312,
|
|
"regularization/lipschitz_norm": 1566.7181396484375,
|
|
"regularization/mmd": 0.22262363135814667,
|
|
"regularization/rkhs_norm": 265.0281677246094,
|
|
"regularization/w1": 1.3160432577133179,
|
|
"rewards/chosen": 0.46457556819609125,
|
|
"rewards/margins": 0.7218770147107427,
|
|
"rewards/rejected": -0.2573014465146514,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 20.832820892333984,
|
|
"kl": 9.469535827636719,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34254809.419152275,
|
|
"logits/rejected": -35309067.94401244,
|
|
"logps/chosen": -460.2936616954474,
|
|
"logps/rejected": -373.69600505443236,
|
|
"loss": 0.6059,
|
|
"loss/base_kto": 3.390723943710327,
|
|
"metrics/advantage_var": 496.0002136230469,
|
|
"regularization/lipschitz_norm": 1483.6629638671875,
|
|
"regularization/mmd": 0.21015730500221252,
|
|
"regularization/rkhs_norm": 250.18728637695312,
|
|
"regularization/w1": 1.246277093887329,
|
|
"rewards/chosen": 0.24943207122467376,
|
|
"rewards/margins": 0.6502812192764088,
|
|
"rewards/rejected": -0.40084914805173505,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 17.37612533569336,
|
|
"kl": 21.6011905670166,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34499173.4562212,
|
|
"logits/rejected": -35013011.73926868,
|
|
"logps/chosen": -483.16815476190476,
|
|
"logps/rejected": -373.6700616057234,
|
|
"loss": 0.6079,
|
|
"loss/base_kto": 3.3115715980529785,
|
|
"metrics/advantage_var": 614.5451049804688,
|
|
"regularization/lipschitz_norm": 1579.8714599609375,
|
|
"regularization/mmd": 0.22479060292243958,
|
|
"regularization/rkhs_norm": 267.6078796386719,
|
|
"regularization/w1": 1.3270920515060425,
|
|
"rewards/chosen": 0.5977829143565189,
|
|
"rewards/margins": 0.7245149203385464,
|
|
"rewards/rejected": -0.12673200598202752,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.475263595581055,
|
|
"kl": 14.524096488952637,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35779786.852614895,
|
|
"logits/rejected": -36290168.70261941,
|
|
"logps/chosen": -485.1234647385103,
|
|
"logps/rejected": -380.91484495377506,
|
|
"loss": 0.5893,
|
|
"loss/base_kto": 3.2896792888641357,
|
|
"metrics/advantage_var": 497.2733459472656,
|
|
"regularization/lipschitz_norm": 1447.8935546875,
|
|
"regularization/mmd": 0.20876596868038177,
|
|
"regularization/rkhs_norm": 248.53091430664062,
|
|
"regularization/w1": 1.2162306308746338,
|
|
"rewards/chosen": 0.3958640121242324,
|
|
"rewards/margins": 0.7231573963647105,
|
|
"rewards/rejected": -0.32729338424047816,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 18.779521942138672,
|
|
"kl": 17.58418846130371,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34910112.0,
|
|
"logits/rejected": -35844006.4,
|
|
"logps/chosen": -489.531591796875,
|
|
"logps/rejected": -379.358154296875,
|
|
"loss": 0.5977,
|
|
"loss/base_kto": 3.2318897247314453,
|
|
"metrics/advantage_var": 584.5302124023438,
|
|
"regularization/lipschitz_norm": 1577.4110107421875,
|
|
"regularization/mmd": 0.22452712059020996,
|
|
"regularization/rkhs_norm": 267.294189453125,
|
|
"regularization/w1": 1.325025200843811,
|
|
"rewards/chosen": 0.5390872478485107,
|
|
"rewards/margins": 0.8077551841735839,
|
|
"rewards/rejected": -0.26866793632507324,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 18.532835006713867,
|
|
"kl": 20.35678482055664,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -35310502.822822824,
|
|
"logits/rejected": -35952519.921824105,
|
|
"logps/chosen": -467.0286692942943,
|
|
"logps/rejected": -371.42536135993487,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.258364200592041,
|
|
"metrics/advantage_var": 617.3034057617188,
|
|
"regularization/lipschitz_norm": 1577.9873046875,
|
|
"regularization/mmd": 0.22676070034503937,
|
|
"regularization/rkhs_norm": 269.9532165527344,
|
|
"regularization/w1": 1.3255093097686768,
|
|
"rewards/chosen": 0.6568281385633681,
|
|
"rewards/margins": 0.7860093946867649,
|
|
"rewards/rejected": -0.1291812561233968,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 22.125225067138672,
|
|
"kl": 18.209014892578125,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -35537974.57053292,
|
|
"logits/rejected": -37848507.41433022,
|
|
"logps/chosen": -481.35883620689657,
|
|
"logps/rejected": -379.90732087227417,
|
|
"loss": 0.5996,
|
|
"loss/base_kto": 3.314927816390991,
|
|
"metrics/advantage_var": 557.0399780273438,
|
|
"regularization/lipschitz_norm": 1506.515869140625,
|
|
"regularization/mmd": 0.21605709195137024,
|
|
"regularization/rkhs_norm": 257.2108459472656,
|
|
"regularization/w1": 1.2654732465744019,
|
|
"rewards/chosen": 0.5158357530552018,
|
|
"rewards/margins": 0.687892860285415,
|
|
"rewards/rejected": -0.1720571072302132,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 23.97228240966797,
|
|
"kl": 18.614089965820312,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34027892.36363637,
|
|
"logits/rejected": -36347170.48494453,
|
|
"logps/chosen": -481.3488058551618,
|
|
"logps/rejected": -357.94683538034866,
|
|
"loss": 0.5962,
|
|
"loss/base_kto": 3.295128345489502,
|
|
"metrics/advantage_var": 504.3421325683594,
|
|
"regularization/lipschitz_norm": 1502.4940185546875,
|
|
"regularization/mmd": 0.21207447350025177,
|
|
"regularization/rkhs_norm": 252.46963500976562,
|
|
"regularization/w1": 1.2620948553085327,
|
|
"rewards/chosen": 0.5352349656021282,
|
|
"rewards/margins": 0.7317881188079152,
|
|
"rewards/rejected": -0.19655315320578695,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.866966247558594,
|
|
"kl": 12.999682426452637,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34775377.05329154,
|
|
"logits/rejected": -34788406.23052959,
|
|
"logps/chosen": -470.23912617554856,
|
|
"logps/rejected": -366.16737246884736,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.294736623764038,
|
|
"metrics/advantage_var": 538.5147705078125,
|
|
"regularization/lipschitz_norm": 1535.3551025390625,
|
|
"regularization/mmd": 0.2138172686100006,
|
|
"regularization/rkhs_norm": 254.5443878173828,
|
|
"regularization/w1": 1.2896982431411743,
|
|
"rewards/chosen": 0.4957161547621963,
|
|
"rewards/margins": 0.7595857285276509,
|
|
"rewards/rejected": -0.2638695737654546,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 19.279701232910156,
|
|
"kl": 11.23679256439209,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34983882.891410045,
|
|
"logits/rejected": -35902595.28205128,
|
|
"logps/chosen": -498.36487034035656,
|
|
"logps/rejected": -382.80554769984917,
|
|
"loss": 0.6135,
|
|
"loss/base_kto": 3.3142383098602295,
|
|
"metrics/advantage_var": 668.0875854492188,
|
|
"regularization/lipschitz_norm": 1626.1376953125,
|
|
"regularization/mmd": 0.2277829647064209,
|
|
"regularization/rkhs_norm": 271.17022705078125,
|
|
"regularization/w1": 1.3659557104110718,
|
|
"rewards/chosen": 0.3200659852360211,
|
|
"rewards/margins": 0.6886985362882929,
|
|
"rewards/rejected": -0.3686325510522719,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.15862274169922,
|
|
"kl": 7.509047985076904,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -35984176.58692185,
|
|
"logits/rejected": -35629724.03062787,
|
|
"logps/chosen": -502.9591307814992,
|
|
"logps/rejected": -387.4928215926493,
|
|
"loss": 0.61,
|
|
"loss/base_kto": 3.325568914413452,
|
|
"metrics/advantage_var": 557.3280639648438,
|
|
"regularization/lipschitz_norm": 1591.0609130859375,
|
|
"regularization/mmd": 0.2181168645620346,
|
|
"regularization/rkhs_norm": 259.6629333496094,
|
|
"regularization/w1": 1.3364911079406738,
|
|
"rewards/chosen": 0.27292303301310805,
|
|
"rewards/margins": 0.7358475404696787,
|
|
"rewards/rejected": -0.46292450745657066,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 20.61990737915039,
|
|
"kl": 9.741283416748047,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33348244.440310076,
|
|
"logits/rejected": -34671769.1968504,
|
|
"logps/chosen": -476.7390503875969,
|
|
"logps/rejected": -396.48228346456693,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.1920666694641113,
|
|
"metrics/advantage_var": 586.7437744140625,
|
|
"regularization/lipschitz_norm": 1600.35107421875,
|
|
"regularization/mmd": 0.22914905846118927,
|
|
"regularization/rkhs_norm": 272.7965087890625,
|
|
"regularization/w1": 1.3442949056625366,
|
|
"rewards/chosen": 0.440731078155281,
|
|
"rewards/margins": 0.9006241560957338,
|
|
"rewards/rejected": -0.45989307794045275,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 29.126808166503906,
|
|
"kl": 15.4036226272583,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34358231.93740219,
|
|
"logits/rejected": -34454551.9625585,
|
|
"logps/chosen": -477.675029342723,
|
|
"logps/rejected": -366.43540366614667,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.3354976177215576,
|
|
"metrics/advantage_var": 528.1583862304688,
|
|
"regularization/lipschitz_norm": 1512.0611572265625,
|
|
"regularization/mmd": 0.21325476467609406,
|
|
"regularization/rkhs_norm": 253.87474060058594,
|
|
"regularization/w1": 1.2701313495635986,
|
|
"rewards/chosen": 0.4657632890441608,
|
|
"rewards/margins": 0.7195094288799886,
|
|
"rewards/rejected": -0.25374613983582783,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 17.753704071044922,
|
|
"kl": 11.226771354675293,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35167465.6,
|
|
"logits/rejected": -34664812.8,
|
|
"logps/chosen": -489.473046875,
|
|
"logps/rejected": -367.453662109375,
|
|
"loss": 0.6079,
|
|
"loss/base_kto": 3.330612897872925,
|
|
"metrics/advantage_var": 553.4308471679688,
|
|
"regularization/lipschitz_norm": 1566.8941650390625,
|
|
"regularization/mmd": 0.2161492556333542,
|
|
"regularization/rkhs_norm": 257.320556640625,
|
|
"regularization/w1": 1.3161910772323608,
|
|
"rewards/chosen": 0.34350905418395994,
|
|
"rewards/margins": 0.7042253494262696,
|
|
"rewards/rejected": -0.3607162952423096,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 19.2286376953125,
|
|
"kl": 13.970544815063477,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35213343.80124223,
|
|
"logits/rejected": -34739705.559748426,
|
|
"logps/chosen": -474.69322593167703,
|
|
"logps/rejected": -381.9266411163522,
|
|
"loss": 0.6034,
|
|
"loss/base_kto": 3.3423850536346436,
|
|
"metrics/advantage_var": 535.233642578125,
|
|
"regularization/lipschitz_norm": 1512.7137451171875,
|
|
"regularization/mmd": 0.21403108537197113,
|
|
"regularization/rkhs_norm": 254.7988739013672,
|
|
"regularization/w1": 1.2706794738769531,
|
|
"rewards/chosen": 0.48326257741229134,
|
|
"rewards/margins": 0.7013732197554886,
|
|
"rewards/rejected": -0.21811064234319724,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 16.981948852539062,
|
|
"kl": 11.745482444763184,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -33234241.37846154,
|
|
"logits/rejected": -34350336.81269842,
|
|
"logps/chosen": -496.12461538461537,
|
|
"logps/rejected": -365.1012400793651,
|
|
"loss": 0.6087,
|
|
"loss/base_kto": 3.3179357051849365,
|
|
"metrics/advantage_var": 564.701171875,
|
|
"regularization/lipschitz_norm": 1581.0506591796875,
|
|
"regularization/mmd": 0.22370798885822296,
|
|
"regularization/rkhs_norm": 266.3190612792969,
|
|
"regularization/w1": 1.3280826807022095,
|
|
"rewards/chosen": 0.46417987530048077,
|
|
"rewards/margins": 0.7207178546686603,
|
|
"rewards/rejected": -0.25653797936817957,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 15.052535057067871,
|
|
"kl": 12.888473510742188,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35564059.80830671,
|
|
"logits/rejected": -36397773.113149844,
|
|
"logps/chosen": -480.6227036741214,
|
|
"logps/rejected": -367.5986716360856,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.300450086593628,
|
|
"metrics/advantage_var": 530.1325073242188,
|
|
"regularization/lipschitz_norm": 1514.3353271484375,
|
|
"regularization/mmd": 0.21618881821632385,
|
|
"regularization/rkhs_norm": 257.3676452636719,
|
|
"regularization/w1": 1.2720415592193604,
|
|
"rewards/chosen": 0.4311969988643171,
|
|
"rewards/margins": 0.7384272104080624,
|
|
"rewards/rejected": -0.30723021154374525,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 17.960697174072266,
|
|
"kl": 13.118896484375,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34243528.26127527,
|
|
"logits/rejected": -34907854.568288855,
|
|
"logps/chosen": -496.6147939346812,
|
|
"logps/rejected": -372.7398940345369,
|
|
"loss": 0.5958,
|
|
"loss/base_kto": 3.3089871406555176,
|
|
"metrics/advantage_var": 647.0138549804688,
|
|
"regularization/lipschitz_norm": 1477.0413818359375,
|
|
"regularization/mmd": 0.21669898927211761,
|
|
"regularization/rkhs_norm": 257.9750061035156,
|
|
"regularization/w1": 1.2407147884368896,
|
|
"rewards/chosen": 0.4499052293934681,
|
|
"rewards/margins": 0.7542690131055707,
|
|
"rewards/rejected": -0.30436378371210265,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 15.313211441040039,
|
|
"kl": 14.095782279968262,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34079965.788018435,
|
|
"logits/rejected": -35523373.320574164,
|
|
"logps/chosen": -493.5922619047619,
|
|
"logps/rejected": -363.98639354066984,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.15490460395813,
|
|
"metrics/advantage_var": 464.8349609375,
|
|
"regularization/lipschitz_norm": 1358.8924560546875,
|
|
"regularization/mmd": 0.20710833370685577,
|
|
"regularization/rkhs_norm": 246.55752563476562,
|
|
"regularization/w1": 1.1414695978164673,
|
|
"rewards/chosen": 0.597113777781778,
|
|
"rewards/margins": 0.8915258096703595,
|
|
"rewards/rejected": -0.29441203188858156,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 21.99187660217285,
|
|
"kl": 13.664057731628418,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -33746682.08320493,
|
|
"logits/rejected": -34160237.540412046,
|
|
"logps/chosen": -476.7251540832049,
|
|
"logps/rejected": -405.28174524564184,
|
|
"loss": 0.5612,
|
|
"loss/base_kto": 3.128077507019043,
|
|
"metrics/advantage_var": 471.5809631347656,
|
|
"regularization/lipschitz_norm": 1379.253173828125,
|
|
"regularization/mmd": 0.20275621116161346,
|
|
"regularization/rkhs_norm": 241.37646484375,
|
|
"regularization/w1": 1.1585725545883179,
|
|
"rewards/chosen": 0.5500277620618259,
|
|
"rewards/margins": 0.9302366677005071,
|
|
"rewards/rejected": -0.38020890563868115,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.422464370727539,
|
|
"kl": 11.592644691467285,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -32613800.967840735,
|
|
"logits/rejected": -34250471.09409888,
|
|
"logps/chosen": -493.29622894333846,
|
|
"logps/rejected": -389.38392145135566,
|
|
"loss": 0.5569,
|
|
"loss/base_kto": 3.191281795501709,
|
|
"metrics/advantage_var": 423.52618408203125,
|
|
"regularization/lipschitz_norm": 1277.8675537109375,
|
|
"regularization/mmd": 0.19023196399211884,
|
|
"regularization/rkhs_norm": 226.4665985107422,
|
|
"regularization/w1": 1.0734087228775024,
|
|
"rewards/chosen": 0.4662266215723105,
|
|
"rewards/margins": 0.8378979179551355,
|
|
"rewards/rejected": -0.371671296382825,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 18.951311111450195,
|
|
"kl": 16.234603881835938,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35214773.386934675,
|
|
"logits/rejected": -35807537.85065886,
|
|
"logps/chosen": -480.63897613065325,
|
|
"logps/rejected": -403.58597181551977,
|
|
"loss": 0.5609,
|
|
"loss/base_kto": 3.1413140296936035,
|
|
"metrics/advantage_var": 453.907470703125,
|
|
"regularization/lipschitz_norm": 1363.2835693359375,
|
|
"regularization/mmd": 0.2005116045475006,
|
|
"regularization/rkhs_norm": 238.7042999267578,
|
|
"regularization/w1": 1.1451581716537476,
|
|
"rewards/chosen": 0.6022577589281041,
|
|
"rewards/margins": 0.8902514958095669,
|
|
"rewards/rejected": -0.28799373688146274,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 14.07236385345459,
|
|
"kl": 5.281290531158447,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -32559478.712933753,
|
|
"logits/rejected": -34528284.53250774,
|
|
"logps/chosen": -480.9516462933754,
|
|
"logps/rejected": -366.27138157894734,
|
|
"loss": 0.5631,
|
|
"loss/base_kto": 3.185199499130249,
|
|
"metrics/advantage_var": 424.0130310058594,
|
|
"regularization/lipschitz_norm": 1336.6717529296875,
|
|
"regularization/mmd": 0.19682419300079346,
|
|
"regularization/rkhs_norm": 234.3145294189453,
|
|
"regularization/w1": 1.1228042840957642,
|
|
"rewards/chosen": 0.3594147113595475,
|
|
"rewards/margins": 0.8819243541583817,
|
|
"rewards/rejected": -0.5225096427988342,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 15.666121482849121,
|
|
"kl": 9.165144920349121,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34367691.18740158,
|
|
"logits/rejected": -34045701.159689926,
|
|
"logps/chosen": -480.58125,
|
|
"logps/rejected": -380.47873062015503,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.1628100872039795,
|
|
"metrics/advantage_var": 443.880859375,
|
|
"regularization/lipschitz_norm": 1340.9677734375,
|
|
"regularization/mmd": 0.19949303567409515,
|
|
"regularization/rkhs_norm": 237.49169921875,
|
|
"regularization/w1": 1.1264129877090454,
|
|
"rewards/chosen": 0.421259842519685,
|
|
"rewards/margins": 0.892059157033372,
|
|
"rewards/rejected": -0.470799314513687,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 11.909934043884277,
|
|
"kl": 10.994178771972656,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34517034.986506745,
|
|
"logits/rejected": -36100747.48450245,
|
|
"logps/chosen": -483.49072338830587,
|
|
"logps/rejected": -373.2572899673736,
|
|
"loss": 0.5589,
|
|
"loss/base_kto": 3.168639898300171,
|
|
"metrics/advantage_var": 424.2437438964844,
|
|
"regularization/lipschitz_norm": 1316.8201904296875,
|
|
"regularization/mmd": 0.1962016075849533,
|
|
"regularization/rkhs_norm": 233.57334899902344,
|
|
"regularization/w1": 1.1061289310455322,
|
|
"rewards/chosen": 0.5468290634955959,
|
|
"rewards/margins": 0.8976001227154292,
|
|
"rewards/rejected": -0.3507710592198333,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 24.540315628051758,
|
|
"kl": 18.536861419677734,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34665291.86687797,
|
|
"logits/rejected": -34876985.590138674,
|
|
"logps/chosen": -478.4107567353407,
|
|
"logps/rejected": -355.77029564714945,
|
|
"loss": 0.5553,
|
|
"loss/base_kto": 3.1770870685577393,
|
|
"metrics/advantage_var": 387.1358337402344,
|
|
"regularization/lipschitz_norm": 1281.3841552734375,
|
|
"regularization/mmd": 0.18883901834487915,
|
|
"regularization/rkhs_norm": 224.808349609375,
|
|
"regularization/w1": 1.0763626098632812,
|
|
"rewards/chosen": 0.5488640593273326,
|
|
"rewards/margins": 0.8243426163320995,
|
|
"rewards/rejected": -0.2754785570047669,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 14.799328804016113,
|
|
"kl": 13.781950950622559,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -32924737.908948194,
|
|
"logits/rejected": -33757935.67651633,
|
|
"logps/chosen": -454.45731946624807,
|
|
"logps/rejected": -356.90156006998444,
|
|
"loss": 0.551,
|
|
"loss/base_kto": 3.1599836349487305,
|
|
"metrics/advantage_var": 404.5981750488281,
|
|
"regularization/lipschitz_norm": 1257.998046875,
|
|
"regularization/mmd": 0.19092433154582977,
|
|
"regularization/rkhs_norm": 227.29087829589844,
|
|
"regularization/w1": 1.056718349456787,
|
|
"rewards/chosen": 0.5308890589739501,
|
|
"rewards/margins": 0.8707831178896772,
|
|
"rewards/rejected": -0.33989405891572705,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 27.282676696777344,
|
|
"kl": 12.126872062683105,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -32593359.698113207,
|
|
"logits/rejected": -35721406.80745342,
|
|
"logps/chosen": -472.6410672169811,
|
|
"logps/rejected": -374.76669254658384,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.2186203002929688,
|
|
"metrics/advantage_var": 421.0472106933594,
|
|
"regularization/lipschitz_norm": 1297.7713623046875,
|
|
"regularization/mmd": 0.19455288350582123,
|
|
"regularization/rkhs_norm": 231.610595703125,
|
|
"regularization/w1": 1.0901278257369995,
|
|
"rewards/chosen": 0.45224627608773094,
|
|
"rewards/margins": 0.8007931512772811,
|
|
"rewards/rejected": -0.3485468751895502,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 18.43590545654297,
|
|
"kl": 11.489155769348145,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -33549320.13990461,
|
|
"logits/rejected": -35002223.287250385,
|
|
"logps/chosen": -492.8347575516693,
|
|
"logps/rejected": -372.88973694316434,
|
|
"loss": 0.5561,
|
|
"loss/base_kto": 3.189197301864624,
|
|
"metrics/advantage_var": 434.7616271972656,
|
|
"regularization/lipschitz_norm": 1268.9654541015625,
|
|
"regularization/mmd": 0.19366741180419922,
|
|
"regularization/rkhs_norm": 230.5564422607422,
|
|
"regularization/w1": 1.065930962562561,
|
|
"rewards/chosen": 0.49731202724438595,
|
|
"rewards/margins": 0.8479111843404401,
|
|
"rewards/rejected": -0.35059915709605416,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 17.76285171508789,
|
|
"kl": 12.488009452819824,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34947078.34055728,
|
|
"logits/rejected": -33757345.514195584,
|
|
"logps/chosen": -470.2392608359133,
|
|
"logps/rejected": -391.8022969242902,
|
|
"loss": 0.5606,
|
|
"loss/base_kto": 3.1847665309906006,
|
|
"metrics/advantage_var": 418.9596862792969,
|
|
"regularization/lipschitz_norm": 1315.7283935546875,
|
|
"regularization/mmd": 0.19452837109565735,
|
|
"regularization/rkhs_norm": 231.58140563964844,
|
|
"regularization/w1": 1.105211853981018,
|
|
"rewards/chosen": 0.48922153154024767,
|
|
"rewards/margins": 0.8552225523869305,
|
|
"rewards/rejected": -0.3660010208466828,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 18.277315139770508,
|
|
"kl": 9.576740264892578,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -33146442.926829267,
|
|
"logits/rejected": -35496228.102564104,
|
|
"logps/chosen": -456.916396722561,
|
|
"logps/rejected": -373.75631009615387,
|
|
"loss": 0.5539,
|
|
"loss/base_kto": 3.2025070190429688,
|
|
"metrics/advantage_var": 392.09246826171875,
|
|
"regularization/lipschitz_norm": 1240.5635986328125,
|
|
"regularization/mmd": 0.18656037747859955,
|
|
"regularization/rkhs_norm": 222.09568786621094,
|
|
"regularization/w1": 1.0420734882354736,
|
|
"rewards/chosen": 0.5252840926007527,
|
|
"rewards/margins": 0.8435238181538252,
|
|
"rewards/rejected": -0.3182397255530724,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 14.79880142211914,
|
|
"kl": 14.9078369140625,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35006041.987878785,
|
|
"logits/rejected": -36812466.37419355,
|
|
"logps/chosen": -502.13442234848486,
|
|
"logps/rejected": -362.98939012096776,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.174055576324463,
|
|
"metrics/advantage_var": 411.0618591308594,
|
|
"regularization/lipschitz_norm": 1300.8685302734375,
|
|
"regularization/mmd": 0.191409632563591,
|
|
"regularization/rkhs_norm": 227.8685760498047,
|
|
"regularization/w1": 1.0927295684814453,
|
|
"rewards/chosen": 0.5695584846265388,
|
|
"rewards/margins": 0.8604397338389884,
|
|
"rewards/rejected": -0.2908812492124496,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 21.873613357543945,
|
|
"kl": 17.760068893432617,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33957355.64525994,
|
|
"logits/rejected": -35875244.57507987,
|
|
"logps/chosen": -499.5922687308869,
|
|
"logps/rejected": -394.015375399361,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.165364980697632,
|
|
"metrics/advantage_var": 412.9040222167969,
|
|
"regularization/lipschitz_norm": 1346.7132568359375,
|
|
"regularization/mmd": 0.1925448179244995,
|
|
"regularization/rkhs_norm": 229.2200164794922,
|
|
"regularization/w1": 1.1312392950057983,
|
|
"rewards/chosen": 0.6139480170853641,
|
|
"rewards/margins": 0.8471430690646972,
|
|
"rewards/rejected": -0.23319505197933307,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 14.800293922424316,
|
|
"kl": 12.878097534179688,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34800084.29268292,
|
|
"logits/rejected": -36039614.35897436,
|
|
"logps/chosen": -488.6559641768293,
|
|
"logps/rejected": -348.6477864583333,
|
|
"loss": 0.5617,
|
|
"loss/base_kto": 3.1702327728271484,
|
|
"metrics/advantage_var": 449.072265625,
|
|
"regularization/lipschitz_norm": 1340.468994140625,
|
|
"regularization/mmd": 0.1974276602268219,
|
|
"regularization/rkhs_norm": 235.0329132080078,
|
|
"regularization/w1": 1.1259939670562744,
|
|
"rewards/chosen": 0.5669898056402439,
|
|
"rewards/margins": 0.8600389577210732,
|
|
"rewards/rejected": -0.2930491520808293,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 16.028493881225586,
|
|
"kl": 14.466745376586914,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33661005.283018865,
|
|
"logits/rejected": -35080840.745341614,
|
|
"logps/chosen": -471.712902908805,
|
|
"logps/rejected": -365.0151882763975,
|
|
"loss": 0.5515,
|
|
"loss/base_kto": 3.158076047897339,
|
|
"metrics/advantage_var": 391.3356628417969,
|
|
"regularization/lipschitz_norm": 1269.25,
|
|
"regularization/mmd": 0.18780864775180817,
|
|
"regularization/rkhs_norm": 223.5817413330078,
|
|
"regularization/w1": 1.0661700963974,
|
|
"rewards/chosen": 0.5329361441750197,
|
|
"rewards/margins": 0.8511174367858989,
|
|
"rewards/rejected": -0.3181812926108793,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 16.022851943969727,
|
|
"kl": 12.29040813446045,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34109908.743471585,
|
|
"logits/rejected": -34917136.68680445,
|
|
"logps/chosen": -480.93092357910905,
|
|
"logps/rejected": -385.74175278219394,
|
|
"loss": 0.5583,
|
|
"loss/base_kto": 3.20527720451355,
|
|
"metrics/advantage_var": 424.3740234375,
|
|
"regularization/lipschitz_norm": 1272.1112060546875,
|
|
"regularization/mmd": 0.1927184909582138,
|
|
"regularization/rkhs_norm": 229.4268035888672,
|
|
"regularization/w1": 1.0685734748840332,
|
|
"rewards/chosen": 0.47949368759600614,
|
|
"rewards/margins": 0.8281711848169173,
|
|
"rewards/rejected": -0.3486774972209112,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 15.451380729675293,
|
|
"kl": 12.152153015136719,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -32477039.47147147,
|
|
"logits/rejected": -35721432.80781759,
|
|
"logps/chosen": -499.37537537537537,
|
|
"logps/rejected": -379.37296416938113,
|
|
"loss": 0.5563,
|
|
"loss/base_kto": 3.201504945755005,
|
|
"metrics/advantage_var": 432.906982421875,
|
|
"regularization/lipschitz_norm": 1258.7347412109375,
|
|
"regularization/mmd": 0.19174246490001678,
|
|
"regularization/rkhs_norm": 228.2648468017578,
|
|
"regularization/w1": 1.0573372840881348,
|
|
"rewards/chosen": 0.5399559410484703,
|
|
"rewards/margins": 0.8619392038964702,
|
|
"rewards/rejected": -0.3219832628479998,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 13.201750755310059,
|
|
"kl": 17.555028915405273,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -33463276.679245282,
|
|
"logits/rejected": -35668244.67080745,
|
|
"logps/chosen": -460.61025943396226,
|
|
"logps/rejected": -373.3250679347826,
|
|
"loss": 0.5499,
|
|
"loss/base_kto": 3.1087777614593506,
|
|
"metrics/advantage_var": 461.3937683105469,
|
|
"regularization/lipschitz_norm": 1298.123291015625,
|
|
"regularization/mmd": 0.2002681940793991,
|
|
"regularization/rkhs_norm": 238.41455078125,
|
|
"regularization/w1": 1.090423583984375,
|
|
"rewards/chosen": 0.6621928664873231,
|
|
"rewards/margins": 0.9513335559343825,
|
|
"rewards/rejected": -0.2891406894470594,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 14.60839557647705,
|
|
"kl": 14.942126274108887,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -32976380.738853503,
|
|
"logits/rejected": -33695065.521472394,
|
|
"logps/chosen": -459.99034633757964,
|
|
"logps/rejected": -356.320959547546,
|
|
"loss": 0.5559,
|
|
"loss/base_kto": 3.173603057861328,
|
|
"metrics/advantage_var": 412.01416015625,
|
|
"regularization/lipschitz_norm": 1288.9000244140625,
|
|
"regularization/mmd": 0.1908697485923767,
|
|
"regularization/rkhs_norm": 227.2258758544922,
|
|
"regularization/w1": 1.0826759338378906,
|
|
"rewards/chosen": 0.6032223063669387,
|
|
"rewards/margins": 0.8704369819894224,
|
|
"rewards/rejected": -0.2672146756224837,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 19.698694229125977,
|
|
"kl": 13.6075439453125,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35279841.55242567,
|
|
"logits/rejected": -36537959.03900156,
|
|
"logps/chosen": -496.9340766823161,
|
|
"logps/rejected": -353.2669656786272,
|
|
"loss": 0.5591,
|
|
"loss/base_kto": 3.1946465969085693,
|
|
"metrics/advantage_var": 411.2864685058594,
|
|
"regularization/lipschitz_norm": 1295.8863525390625,
|
|
"regularization/mmd": 0.18988429009914398,
|
|
"regularization/rkhs_norm": 226.052734375,
|
|
"regularization/w1": 1.088544487953186,
|
|
"rewards/chosen": 0.510437441543794,
|
|
"rewards/margins": 0.8200323204561379,
|
|
"rewards/rejected": -0.309594878912344,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 11.972624778747559,
|
|
"kl": 15.9436674118042,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -35117256.64864865,
|
|
"logits/rejected": -35717030.697674416,
|
|
"logps/chosen": -472.54222972972974,
|
|
"logps/rejected": -411.9827398255814,
|
|
"loss": 0.5542,
|
|
"loss/base_kto": 3.171508550643921,
|
|
"metrics/advantage_var": 418.6213073730469,
|
|
"regularization/lipschitz_norm": 1275.0072021484375,
|
|
"regularization/mmd": 0.1913410872220993,
|
|
"regularization/rkhs_norm": 227.78701782226562,
|
|
"regularization/w1": 1.071006178855896,
|
|
"rewards/chosen": 0.4782499364904455,
|
|
"rewards/margins": 0.8332488577295443,
|
|
"rewards/rejected": -0.3549989212390988,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 20.721656799316406,
|
|
"kl": 12.98636531829834,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34533147.0310559,
|
|
"logits/rejected": -35465422.08805031,
|
|
"logps/chosen": -485.6110248447205,
|
|
"logps/rejected": -376.31274567610063,
|
|
"loss": 0.5598,
|
|
"loss/base_kto": 3.2010269165039062,
|
|
"metrics/advantage_var": 403.4350280761719,
|
|
"regularization/lipschitz_norm": 1294.6414794921875,
|
|
"regularization/mmd": 0.18953049182891846,
|
|
"regularization/rkhs_norm": 225.6315460205078,
|
|
"regularization/w1": 1.0874987840652466,
|
|
"rewards/chosen": 0.5154436478703659,
|
|
"rewards/margins": 0.8386137399912262,
|
|
"rewards/rejected": -0.32317009212086034,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 14.437726020812988,
|
|
"kl": 14.29247760772705,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34227536.95024077,
|
|
"logits/rejected": -35142339.60426179,
|
|
"logps/chosen": -488.5439907704655,
|
|
"logps/rejected": -385.6804128614916,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.1776986122131348,
|
|
"metrics/advantage_var": 429.72412109375,
|
|
"regularization/lipschitz_norm": 1276.6785888671875,
|
|
"regularization/mmd": 0.18969428539276123,
|
|
"regularization/rkhs_norm": 225.8265380859375,
|
|
"regularization/w1": 1.072409987449646,
|
|
"rewards/chosen": 0.49106819556957265,
|
|
"rewards/margins": 0.8367201792400207,
|
|
"rewards/rejected": -0.3456519836704481,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 14.496883392333984,
|
|
"kl": 12.725996971130371,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -35495131.99381762,
|
|
"logits/rejected": -35898097.845181674,
|
|
"logps/chosen": -513.0717735703246,
|
|
"logps/rejected": -367.4614435229068,
|
|
"loss": 0.5534,
|
|
"loss/base_kto": 3.196915626525879,
|
|
"metrics/advantage_var": 394.6297302246094,
|
|
"regularization/lipschitz_norm": 1241.37353515625,
|
|
"regularization/mmd": 0.18732020258903503,
|
|
"regularization/rkhs_norm": 223.000244140625,
|
|
"regularization/w1": 1.0427536964416504,
|
|
"rewards/chosen": 0.5411957588962277,
|
|
"rewards/margins": 0.848693021474291,
|
|
"rewards/rejected": -0.3074972625780633,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 20.6961669921875,
|
|
"kl": 14.62802791595459,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -36099440.0,
|
|
"logits/rejected": -35340838.4,
|
|
"logps/chosen": -486.097802734375,
|
|
"logps/rejected": -376.86513671875,
|
|
"loss": 0.5612,
|
|
"loss/base_kto": 3.166748046875,
|
|
"metrics/advantage_var": 416.572509765625,
|
|
"regularization/lipschitz_norm": 1346.7005615234375,
|
|
"regularization/mmd": 0.19191506505012512,
|
|
"regularization/rkhs_norm": 228.47032165527344,
|
|
"regularization/w1": 1.1312285661697388,
|
|
"rewards/chosen": 0.5315794944763184,
|
|
"rewards/margins": 0.8421450614929199,
|
|
"rewards/rejected": -0.31056556701660154,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 15.3798828125,
|
|
"kl": 13.744409561157227,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33981523.29186603,
|
|
"logits/rejected": -35471786.53598775,
|
|
"logps/chosen": -489.9465709728868,
|
|
"logps/rejected": -364.75753732771824,
|
|
"loss": 0.5529,
|
|
"loss/base_kto": 3.161714553833008,
|
|
"metrics/advantage_var": 379.2135314941406,
|
|
"regularization/lipschitz_norm": 1281.203369140625,
|
|
"regularization/mmd": 0.185001939535141,
|
|
"regularization/rkhs_norm": 220.2404022216797,
|
|
"regularization/w1": 1.076210856437683,
|
|
"rewards/chosen": 0.5116783519300737,
|
|
"rewards/margins": 0.8561211430144917,
|
|
"rewards/rejected": -0.34444279108441805,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 16.079286575317383,
|
|
"kl": 10.577516555786133,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -32599185.362776026,
|
|
"logits/rejected": -35181866.006191954,
|
|
"logps/chosen": -458.7748915615142,
|
|
"logps/rejected": -383.5875822368421,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.205808401107788,
|
|
"metrics/advantage_var": 409.4687194824219,
|
|
"regularization/lipschitz_norm": 1276.6395263671875,
|
|
"regularization/mmd": 0.1913437843322754,
|
|
"regularization/rkhs_norm": 227.7902374267578,
|
|
"regularization/w1": 1.0723772048950195,
|
|
"rewards/chosen": 0.4623446780424389,
|
|
"rewards/margins": 0.8323916184237518,
|
|
"rewards/rejected": -0.37004694038131286,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 15.495466232299805,
|
|
"kl": 13.636786460876465,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35064352.0,
|
|
"logits/rejected": -35054684.8,
|
|
"logps/chosen": -469.39677734375,
|
|
"logps/rejected": -400.6271484375,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.1601617336273193,
|
|
"metrics/advantage_var": 404.0318298339844,
|
|
"regularization/lipschitz_norm": 1295.0360107421875,
|
|
"regularization/mmd": 0.19014187157154083,
|
|
"regularization/rkhs_norm": 226.359375,
|
|
"regularization/w1": 1.0878301858901978,
|
|
"rewards/chosen": 0.5035799980163574,
|
|
"rewards/margins": 0.8659909248352051,
|
|
"rewards/rejected": -0.3624109268188477,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 16.31978988647461,
|
|
"kl": 12.713685989379883,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -33840813.94871795,
|
|
"logits/rejected": -35110381.26829268,
|
|
"logps/chosen": -491.75666065705127,
|
|
"logps/rejected": -377.2046017530488,
|
|
"loss": 0.5516,
|
|
"loss/base_kto": 3.1769847869873047,
|
|
"metrics/advantage_var": 388.0572204589844,
|
|
"regularization/lipschitz_norm": 1247.1806640625,
|
|
"regularization/mmd": 0.18854664266109467,
|
|
"regularization/rkhs_norm": 224.4602813720703,
|
|
"regularization/w1": 1.0476316213607788,
|
|
"rewards/chosen": 0.49259009728064906,
|
|
"rewards/margins": 0.8505412382659053,
|
|
"rewards/rejected": -0.35795114098525627,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 14.2222318649292,
|
|
"kl": 11.682733535766602,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -33261616.301886793,
|
|
"logits/rejected": -34935305.54037267,
|
|
"logps/chosen": -471.3539701257862,
|
|
"logps/rejected": -372.10078610248445,
|
|
"loss": 0.5589,
|
|
"loss/base_kto": 3.196542739868164,
|
|
"metrics/advantage_var": 416.1800842285156,
|
|
"regularization/lipschitz_norm": 1288.1419677734375,
|
|
"regularization/mmd": 0.1924648880958557,
|
|
"regularization/rkhs_norm": 229.12490844726562,
|
|
"regularization/w1": 1.0820392370224,
|
|
"rewards/chosen": 0.46548020464819184,
|
|
"rewards/margins": 0.849418632685847,
|
|
"rewards/rejected": -0.38393842803765527,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 12.739497184753418,
|
|
"kl": 11.841665267944336,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -33898094.81200632,
|
|
"logits/rejected": -34032302.88717156,
|
|
"logps/chosen": -481.4318720379147,
|
|
"logps/rejected": -380.85872295208657,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.178837299346924,
|
|
"metrics/advantage_var": 413.7232360839844,
|
|
"regularization/lipschitz_norm": 1320.348876953125,
|
|
"regularization/mmd": 0.19114167988300323,
|
|
"regularization/rkhs_norm": 227.5496368408203,
|
|
"regularization/w1": 1.1090930700302124,
|
|
"rewards/chosen": 0.5040864628073164,
|
|
"rewards/margins": 0.8582767526894044,
|
|
"rewards/rejected": -0.354190289882088,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 13.439664840698242,
|
|
"kl": 11.289209365844727,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34617773.51975684,
|
|
"logits/rejected": -33817000.74598071,
|
|
"logps/chosen": -478.82052621580544,
|
|
"logps/rejected": -377.2416097266881,
|
|
"loss": 0.5478,
|
|
"loss/base_kto": 3.2001938819885254,
|
|
"metrics/advantage_var": 361.73919677734375,
|
|
"regularization/lipschitz_norm": 1192.5003662109375,
|
|
"regularization/mmd": 0.18043823540210724,
|
|
"regularization/rkhs_norm": 214.8074188232422,
|
|
"regularization/w1": 1.0017002820968628,
|
|
"rewards/chosen": 0.45874213592622054,
|
|
"rewards/margins": 0.8388894504578474,
|
|
"rewards/rejected": -0.3801473145316268,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 16.273509979248047,
|
|
"kl": 12.528304100036621,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34907047.5855573,
|
|
"logits/rejected": -36295133.76049767,
|
|
"logps/chosen": -494.18514521193094,
|
|
"logps/rejected": -365.4915435458787,
|
|
"loss": 0.5613,
|
|
"loss/base_kto": 3.189305067062378,
|
|
"metrics/advantage_var": 417.52569580078125,
|
|
"regularization/lipschitz_norm": 1320.7908935546875,
|
|
"regularization/mmd": 0.19167792797088623,
|
|
"regularization/rkhs_norm": 228.1880340576172,
|
|
"regularization/w1": 1.109464406967163,
|
|
"rewards/chosen": 0.4879748286033163,
|
|
"rewards/margins": 0.8510843638324668,
|
|
"rewards/rejected": -0.36310953522915046,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 16.54901695251465,
|
|
"kl": 11.749494552612305,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34998600.85214626,
|
|
"logits/rejected": -35260482.06451613,
|
|
"logps/chosen": -481.5027325119237,
|
|
"logps/rejected": -388.9812307987711,
|
|
"loss": 0.5482,
|
|
"loss/base_kto": 3.235278367996216,
|
|
"metrics/advantage_var": 355.49371337890625,
|
|
"regularization/lipschitz_norm": 1157.6878662109375,
|
|
"regularization/mmd": 0.17820025980472565,
|
|
"regularization/rkhs_norm": 212.14317321777344,
|
|
"regularization/w1": 0.9724577069282532,
|
|
"rewards/chosen": 0.45505318694728736,
|
|
"rewards/margins": 0.7979859627699399,
|
|
"rewards/rejected": -0.34293277582265264,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 15.418791770935059,
|
|
"kl": 15.389676094055176,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34669688.47058824,
|
|
"logits/rejected": -35312860.74251497,
|
|
"logps/chosen": -461.4896854575163,
|
|
"logps/rejected": -371.4582241766467,
|
|
"loss": 0.5582,
|
|
"loss/base_kto": 3.1709365844726562,
|
|
"metrics/advantage_var": 422.8125915527344,
|
|
"regularization/lipschitz_norm": 1309.7713623046875,
|
|
"regularization/mmd": 0.1947401612997055,
|
|
"regularization/rkhs_norm": 231.83349609375,
|
|
"regularization/w1": 1.1002079248428345,
|
|
"rewards/chosen": 0.49242710288054026,
|
|
"rewards/margins": 0.8602050357842641,
|
|
"rewards/rejected": -0.3677779329037238,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 15.599148750305176,
|
|
"kl": 10.940035820007324,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34436028.952380955,
|
|
"logits/rejected": -35767602.526315786,
|
|
"logps/chosen": -502.3232886904762,
|
|
"logps/rejected": -359.4698550575658,
|
|
"loss": 0.554,
|
|
"loss/base_kto": 3.1439406871795654,
|
|
"metrics/advantage_var": 425.60870361328125,
|
|
"regularization/lipschitz_norm": 1301.0902099609375,
|
|
"regularization/mmd": 0.19541232287883759,
|
|
"regularization/rkhs_norm": 232.63369750976562,
|
|
"regularization/w1": 1.0929158926010132,
|
|
"rewards/chosen": 0.5357532501220703,
|
|
"rewards/margins": 0.9235553239521228,
|
|
"rewards/rejected": -0.38780207383005244,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_loss": 0.5860486948840152,
|
|
"train_runtime": 11119.0077,
|
|
"train_samples_per_second": 13.33,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|