02510c0957
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed1 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 21.002891540527344,
|
|
"kl": 9.29682445526123,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36806738.55503876,
|
|
"logits/rejected": -37180559.521259844,
|
|
"logps/chosen": -499.56027131782946,
|
|
"logps/rejected": -361.4782972440945,
|
|
"loss": 0.6118,
|
|
"loss/base_kto": 3.9507997035980225,
|
|
"metrics/advantage_var": 208.28306579589844,
|
|
"regularization/lipschitz_norm": 959.6640625,
|
|
"regularization/mmd": 0.13723795115947723,
|
|
"regularization/rkhs_norm": 163.37852478027344,
|
|
"regularization/w1": 0.806117832660675,
|
|
"rewards/chosen": 0.10371240423631299,
|
|
"rewards/margins": 0.04959450762313939,
|
|
"rewards/rejected": 0.0541178966131736,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 23.644874572753906,
|
|
"kl": 6.305409908294678,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35816444.86850153,
|
|
"logits/rejected": -37655712.30670927,
|
|
"logps/chosen": -485.7736525229358,
|
|
"logps/rejected": -375.6683805910543,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.9509849548339844,
|
|
"metrics/advantage_var": 168.64268493652344,
|
|
"regularization/lipschitz_norm": 863.5946655273438,
|
|
"regularization/mmd": 0.12421522289514542,
|
|
"regularization/rkhs_norm": 147.87527465820312,
|
|
"regularization/w1": 0.7254195213317871,
|
|
"rewards/chosen": 0.07522921314297831,
|
|
"rewards/margins": 0.039415747476131205,
|
|
"rewards/rejected": 0.03581346566684711,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 26.063945770263672,
|
|
"kl": 14.707015037536621,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36950103.681957185,
|
|
"logits/rejected": -37692000.51118211,
|
|
"logps/chosen": -468.01920871559633,
|
|
"logps/rejected": -352.55633486421726,
|
|
"loss": 0.5923,
|
|
"loss/base_kto": 3.8738510608673096,
|
|
"metrics/advantage_var": 167.3021697998047,
|
|
"regularization/lipschitz_norm": 879.8822631835938,
|
|
"regularization/mmd": 0.12527818977832794,
|
|
"regularization/rkhs_norm": 149.14068603515625,
|
|
"regularization/w1": 0.7391010522842407,
|
|
"rewards/chosen": 0.25857277966420583,
|
|
"rewards/margins": 0.12181232393243915,
|
|
"rewards/rejected": 0.13676045573176668,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 31.767148971557617,
|
|
"kl": 5.509912014007568,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36536549.276853256,
|
|
"logits/rejected": -37785245.98384491,
|
|
"logps/chosen": -503.0107791225416,
|
|
"logps/rejected": -383.53049273021,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.858879566192627,
|
|
"metrics/advantage_var": 186.58255004882812,
|
|
"regularization/lipschitz_norm": 941.4556884765625,
|
|
"regularization/mmd": 0.13063393533229828,
|
|
"regularization/rkhs_norm": 155.5166015625,
|
|
"regularization/w1": 0.7908228039741516,
|
|
"rewards/chosen": 0.0899936774134095,
|
|
"rewards/margins": 0.13652343498551578,
|
|
"rewards/rejected": -0.046529757572106284,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 27.916942596435547,
|
|
"kl": 18.066015243530273,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35552688.35555556,
|
|
"logits/rejected": -38070876.947692305,
|
|
"logps/chosen": -494.04117063492066,
|
|
"logps/rejected": -359.19771634615387,
|
|
"loss": 0.602,
|
|
"loss/base_kto": 3.890141248703003,
|
|
"metrics/advantage_var": 213.2407684326172,
|
|
"regularization/lipschitz_norm": 938.4644775390625,
|
|
"regularization/mmd": 0.13715031743049622,
|
|
"regularization/rkhs_norm": 163.274169921875,
|
|
"regularization/w1": 0.7883100509643555,
|
|
"rewards/chosen": 0.27566438705202134,
|
|
"rewards/margins": 0.1004450126154228,
|
|
"rewards/rejected": 0.17521937443659855,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 22.59275245666504,
|
|
"kl": 24.412830352783203,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35760988.84370258,
|
|
"logits/rejected": -37916408.16747182,
|
|
"logps/chosen": -480.1171282245827,
|
|
"logps/rejected": -350.6891354669887,
|
|
"loss": 0.5849,
|
|
"loss/base_kto": 3.785731554031372,
|
|
"metrics/advantage_var": 185.63209533691406,
|
|
"regularization/lipschitz_norm": 905.6769409179688,
|
|
"regularization/mmd": 0.13287097215652466,
|
|
"regularization/rkhs_norm": 158.1797332763672,
|
|
"regularization/w1": 0.7607686519622803,
|
|
"rewards/chosen": 0.4051793840839577,
|
|
"rewards/margins": 0.16112382499013223,
|
|
"rewards/rejected": 0.24405555909382548,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 48.02346420288086,
|
|
"kl": 19.847946166992188,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35990689.514195584,
|
|
"logits/rejected": -38065316.85448916,
|
|
"logps/chosen": -488.46746845425866,
|
|
"logps/rejected": -385.6934017027864,
|
|
"loss": 0.5967,
|
|
"loss/base_kto": 3.826545476913452,
|
|
"metrics/advantage_var": 218.2530975341797,
|
|
"regularization/lipschitz_norm": 961.408203125,
|
|
"regularization/mmd": 0.13978762924671173,
|
|
"regularization/rkhs_norm": 166.41384887695312,
|
|
"regularization/w1": 0.8075828552246094,
|
|
"rewards/chosen": 0.3045312057158172,
|
|
"rewards/margins": 0.15000710230903508,
|
|
"rewards/rejected": 0.15452410340678213,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 17.178922653198242,
|
|
"kl": 4.906111240386963,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -34970382.76112027,
|
|
"logits/rejected": -37107091.209509656,
|
|
"logps/chosen": -492.3693369028007,
|
|
"logps/rejected": -384.2996842496285,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.837249755859375,
|
|
"metrics/advantage_var": 183.5966796875,
|
|
"regularization/lipschitz_norm": 924.1876831054688,
|
|
"regularization/mmd": 0.13173113763332367,
|
|
"regularization/rkhs_norm": 156.82278442382812,
|
|
"regularization/w1": 0.7763177156448364,
|
|
"rewards/chosen": -0.018455899724064785,
|
|
"rewards/margins": 0.13749395785452456,
|
|
"rewards/rejected": -0.15594985757858934,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 18.382226943969727,
|
|
"kl": 2.1383161544799805,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36835695.33968254,
|
|
"logits/rejected": -38161975.13846154,
|
|
"logps/chosen": -507.96517857142857,
|
|
"logps/rejected": -377.80182692307693,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.8357181549072266,
|
|
"metrics/advantage_var": 201.1750946044922,
|
|
"regularization/lipschitz_norm": 973.322265625,
|
|
"regularization/mmd": 0.1369176059961319,
|
|
"regularization/rkhs_norm": 162.9971466064453,
|
|
"regularization/w1": 0.8175907135009766,
|
|
"rewards/chosen": -0.07040049235026041,
|
|
"rewards/margins": 0.16253670716897037,
|
|
"rewards/rejected": -0.23293719951923078,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 24.55632209777832,
|
|
"kl": 2.7123076915740967,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36629456.84210526,
|
|
"logits/rejected": -37932028.952380955,
|
|
"logps/chosen": -488.33475534539474,
|
|
"logps/rejected": -393.32059151785717,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.817567825317383,
|
|
"metrics/advantage_var": 223.64614868164062,
|
|
"regularization/lipschitz_norm": 989.0489501953125,
|
|
"regularization/mmd": 0.1424151062965393,
|
|
"regularization/rkhs_norm": 169.5417938232422,
|
|
"regularization/w1": 0.8308011293411255,
|
|
"rewards/chosen": -0.13143524370695414,
|
|
"rewards/margins": 0.15556473122503528,
|
|
"rewards/rejected": -0.2869999749319894,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 19.38735008239746,
|
|
"kl": 2.8487627506256104,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -34710388.658227846,
|
|
"logits/rejected": -36250478.61728395,
|
|
"logps/chosen": -497.3544303797468,
|
|
"logps/rejected": -390.01916956018516,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.7558200359344482,
|
|
"metrics/advantage_var": 226.3386688232422,
|
|
"regularization/lipschitz_norm": 1014.6015625,
|
|
"regularization/mmd": 0.14520350098609924,
|
|
"regularization/rkhs_norm": 172.86131286621094,
|
|
"regularization/w1": 0.8522653579711914,
|
|
"rewards/chosen": -0.060055056704750545,
|
|
"rewards/margins": 0.2504924540781271,
|
|
"rewards/rejected": -0.3105475107828776,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 23.47049331665039,
|
|
"kl": 12.093279838562012,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36153691.32163743,
|
|
"logits/rejected": -37871588.51006711,
|
|
"logps/chosen": -504.3876096491228,
|
|
"logps/rejected": -354.44347734899327,
|
|
"loss": 0.59,
|
|
"loss/base_kto": 3.7200844287872314,
|
|
"metrics/advantage_var": 235.37034606933594,
|
|
"regularization/lipschitz_norm": 1013.8336181640625,
|
|
"regularization/mmd": 0.14831320941448212,
|
|
"regularization/rkhs_norm": 176.56333923339844,
|
|
"regularization/w1": 0.8516203165054321,
|
|
"rewards/chosen": 0.22248893871641995,
|
|
"rewards/margins": 0.2526094522996096,
|
|
"rewards/rejected": -0.03012051358318969,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 23.581783294677734,
|
|
"kl": 14.000227928161621,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35468692.54320987,
|
|
"logits/rejected": -36875348.25316456,
|
|
"logps/chosen": -485.75327932098764,
|
|
"logps/rejected": -350.51725672468353,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.8040168285369873,
|
|
"metrics/advantage_var": 248.74560546875,
|
|
"regularization/lipschitz_norm": 1037.2283935546875,
|
|
"regularization/mmd": 0.14947238564491272,
|
|
"regularization/rkhs_norm": 177.9433135986328,
|
|
"regularization/w1": 0.8712717294692993,
|
|
"rewards/chosen": 0.21057086520724827,
|
|
"rewards/margins": 0.17433021809649032,
|
|
"rewards/rejected": 0.036240647110757945,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 20.44241714477539,
|
|
"kl": 8.439579963684082,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -35922895.55628059,
|
|
"logits/rejected": -38429209.33133433,
|
|
"logps/chosen": -489.45692292006527,
|
|
"logps/rejected": -366.0670446026987,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.7464218139648438,
|
|
"metrics/advantage_var": 222.99551391601562,
|
|
"regularization/lipschitz_norm": 1004.2482299804688,
|
|
"regularization/mmd": 0.14356186985969543,
|
|
"regularization/rkhs_norm": 170.906982421875,
|
|
"regularization/w1": 0.8435684442520142,
|
|
"rewards/chosen": 0.07846951756936302,
|
|
"rewards/margins": 0.21822585323819851,
|
|
"rewards/rejected": -0.1397563356688355,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 16.739639282226562,
|
|
"kl": 7.388824462890625,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35040198.4,
|
|
"logits/rejected": -36427360.0,
|
|
"logps/chosen": -497.217431640625,
|
|
"logps/rejected": -381.1005859375,
|
|
"loss": 0.6029,
|
|
"loss/base_kto": 3.805175542831421,
|
|
"metrics/advantage_var": 236.3822021484375,
|
|
"regularization/lipschitz_norm": 1033.6092529296875,
|
|
"regularization/mmd": 0.14950346946716309,
|
|
"regularization/rkhs_norm": 177.98031616210938,
|
|
"regularization/w1": 0.8682317733764648,
|
|
"rewards/chosen": 0.03671181797981262,
|
|
"rewards/margins": 0.18501196503639222,
|
|
"rewards/rejected": -0.1483001470565796,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 19.930891036987305,
|
|
"kl": 10.909724235534668,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36241877.268588774,
|
|
"logits/rejected": -36504204.98550725,
|
|
"logps/chosen": -457.81847496206376,
|
|
"logps/rejected": -376.1501107085346,
|
|
"loss": 0.5894,
|
|
"loss/base_kto": 3.797499895095825,
|
|
"metrics/advantage_var": 204.89968872070312,
|
|
"regularization/lipschitz_norm": 928.67236328125,
|
|
"regularization/mmd": 0.13729074597358704,
|
|
"regularization/rkhs_norm": 163.4413299560547,
|
|
"regularization/w1": 0.7800847887992859,
|
|
"rewards/chosen": 0.11744685310514273,
|
|
"rewards/margins": 0.17470345035685053,
|
|
"rewards/rejected": -0.05725659725170781,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 16.12029457092285,
|
|
"kl": 11.883231163024902,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36761102.35514019,
|
|
"logits/rejected": -37523456.0,
|
|
"logps/chosen": -474.08041277258565,
|
|
"logps/rejected": -384.4374020376176,
|
|
"loss": 0.5977,
|
|
"loss/base_kto": 3.7662932872772217,
|
|
"metrics/advantage_var": 242.0256805419922,
|
|
"regularization/lipschitz_norm": 1034.9364013671875,
|
|
"regularization/mmd": 0.14618881046772003,
|
|
"regularization/rkhs_norm": 174.0343017578125,
|
|
"regularization/w1": 0.869346559047699,
|
|
"rewards/chosen": 0.2054643482434044,
|
|
"rewards/margins": 0.22209540588699583,
|
|
"rewards/rejected": -0.016631057643591424,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 16.689865112304688,
|
|
"kl": 6.03048849105835,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -37166285.437014,
|
|
"logits/rejected": -37581134.36734694,
|
|
"logps/chosen": -494.3160478227061,
|
|
"logps/rejected": -382.0531053767661,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.7721123695373535,
|
|
"metrics/advantage_var": 227.2195281982422,
|
|
"regularization/lipschitz_norm": 1009.7874145507812,
|
|
"regularization/mmd": 0.1444280594587326,
|
|
"regularization/rkhs_norm": 171.9381866455078,
|
|
"regularization/w1": 0.8482214212417603,
|
|
"rewards/chosen": 0.05406781043828368,
|
|
"rewards/margins": 0.22711620413094705,
|
|
"rewards/rejected": -0.17304839369266337,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 21.190996170043945,
|
|
"kl": 9.208587646484375,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36699593.6,
|
|
"logits/rejected": -38081651.2,
|
|
"logps/chosen": -501.0798828125,
|
|
"logps/rejected": -369.744287109375,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.7995357513427734,
|
|
"metrics/advantage_var": 229.01504516601562,
|
|
"regularization/lipschitz_norm": 984.9951171875,
|
|
"regularization/mmd": 0.14215107262134552,
|
|
"regularization/rkhs_norm": 169.2274627685547,
|
|
"regularization/w1": 0.8273958563804626,
|
|
"rewards/chosen": 0.11079213619232178,
|
|
"rewards/margins": 0.17351986169815065,
|
|
"rewards/rejected": -0.06272772550582886,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 22.635528564453125,
|
|
"kl": 13.041945457458496,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36352973.046548955,
|
|
"logits/rejected": -37737732.28614917,
|
|
"logps/chosen": -495.12720706260035,
|
|
"logps/rejected": -374.9865391933029,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.7488961219787598,
|
|
"metrics/advantage_var": 239.8693389892578,
|
|
"regularization/lipschitz_norm": 1042.74267578125,
|
|
"regularization/mmd": 0.15008819103240967,
|
|
"regularization/rkhs_norm": 178.6764373779297,
|
|
"regularization/w1": 0.8759037852287292,
|
|
"rewards/chosen": 0.13551915935681683,
|
|
"rewards/margins": 0.21548251828961387,
|
|
"rewards/rejected": -0.07996335893279705,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 20.579927444458008,
|
|
"kl": 12.666115760803223,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36121143.5980551,
|
|
"logits/rejected": -36648413.24886878,
|
|
"logps/chosen": -482.49037682333875,
|
|
"logps/rejected": -372.89698812217193,
|
|
"loss": 0.6018,
|
|
"loss/base_kto": 3.7951207160949707,
|
|
"metrics/advantage_var": 251.12315368652344,
|
|
"regularization/lipschitz_norm": 1035.3060302734375,
|
|
"regularization/mmd": 0.14965009689331055,
|
|
"regularization/rkhs_norm": 178.15489196777344,
|
|
"regularization/w1": 0.869657039642334,
|
|
"rewards/chosen": 0.19201514245625254,
|
|
"rewards/margins": 0.17234569742416442,
|
|
"rewards/rejected": 0.019669445032088107,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 22.452388763427734,
|
|
"kl": 3.7606093883514404,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36301097.18611988,
|
|
"logits/rejected": -37192390.14241486,
|
|
"logps/chosen": -497.71855283911674,
|
|
"logps/rejected": -363.8777089783282,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.774768829345703,
|
|
"metrics/advantage_var": 240.28408813476562,
|
|
"regularization/lipschitz_norm": 1007.1629028320312,
|
|
"regularization/mmd": 0.14802759885787964,
|
|
"regularization/rkhs_norm": 176.2233428955078,
|
|
"regularization/w1": 0.8460167050361633,
|
|
"rewards/chosen": -0.104330766840312,
|
|
"rewards/margins": 0.2134261221958843,
|
|
"rewards/rejected": -0.3177568890361963,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 19.246440887451172,
|
|
"kl": 9.679542541503906,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36739738.79069767,
|
|
"logits/rejected": -38059246.631268434,
|
|
"logps/chosen": -476.8813330564784,
|
|
"logps/rejected": -371.85965154867256,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.7439823150634766,
|
|
"metrics/advantage_var": 243.2819061279297,
|
|
"regularization/lipschitz_norm": 1014.6220092773438,
|
|
"regularization/mmd": 0.1491519659757614,
|
|
"regularization/rkhs_norm": 177.56185913085938,
|
|
"regularization/w1": 0.8522824645042419,
|
|
"rewards/chosen": 0.12866262898492656,
|
|
"rewards/margins": 0.2296599123803393,
|
|
"rewards/rejected": -0.10099728339541275,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 23.29594612121582,
|
|
"kl": 3.4871933460235596,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -35390890.79694656,
|
|
"logits/rejected": -35793141.76,
|
|
"logps/chosen": -458.16049618320613,
|
|
"logps/rejected": -371.07275,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.8229825496673584,
|
|
"metrics/advantage_var": 241.88975524902344,
|
|
"regularization/lipschitz_norm": 990.4679565429688,
|
|
"regularization/mmd": 0.14610660076141357,
|
|
"regularization/rkhs_norm": 173.9364471435547,
|
|
"regularization/w1": 0.8319931030273438,
|
|
"rewards/chosen": -0.07053758461056775,
|
|
"rewards/margins": 0.18983951499880725,
|
|
"rewards/rejected": -0.260377099609375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 17.803701400756836,
|
|
"kl": 15.291159629821777,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36197520.47131783,
|
|
"logits/rejected": -35400121.851968504,
|
|
"logps/chosen": -474.8251453488372,
|
|
"logps/rejected": -373.7671505905512,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.7114250659942627,
|
|
"metrics/advantage_var": 276.8833923339844,
|
|
"regularization/lipschitz_norm": 1095.9642333984375,
|
|
"regularization/mmd": 0.15803752839565277,
|
|
"regularization/rkhs_norm": 188.13990783691406,
|
|
"regularization/w1": 0.9206100702285767,
|
|
"rewards/chosen": 0.2920237933018411,
|
|
"rewards/margins": 0.2693102267245004,
|
|
"rewards/rejected": 0.022713566577340673,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 15.237549781799316,
|
|
"kl": 11.103652954101562,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35199852.32329635,
|
|
"logits/rejected": -35573292.96764252,
|
|
"logps/chosen": -449.81418383518223,
|
|
"logps/rejected": -334.2816592835131,
|
|
"loss": 0.5789,
|
|
"loss/base_kto": 3.6592445373535156,
|
|
"metrics/advantage_var": 248.9097137451172,
|
|
"regularization/lipschitz_norm": 983.8563842773438,
|
|
"regularization/mmd": 0.1453792303800583,
|
|
"regularization/rkhs_norm": 173.0705108642578,
|
|
"regularization/w1": 0.8264393210411072,
|
|
"rewards/chosen": 0.20631609401309925,
|
|
"rewards/margins": 0.326690952851667,
|
|
"rewards/rejected": -0.12037485883856774,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 24.42544937133789,
|
|
"kl": 5.859231472015381,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -35706488.038277514,
|
|
"logits/rejected": -36856140.44716692,
|
|
"logps/chosen": -468.8909489633174,
|
|
"logps/rejected": -387.380288093415,
|
|
"loss": 0.6009,
|
|
"loss/base_kto": 3.785578966140747,
|
|
"metrics/advantage_var": 247.8687286376953,
|
|
"regularization/lipschitz_norm": 1037.9766845703125,
|
|
"regularization/mmd": 0.14969880878925323,
|
|
"regularization/rkhs_norm": 178.21286010742188,
|
|
"regularization/w1": 0.8719003796577454,
|
|
"rewards/chosen": -0.007102211126300137,
|
|
"rewards/margins": 0.22319427579105247,
|
|
"rewards/rejected": -0.2302964869173526,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 23.30228614807129,
|
|
"kl": 6.485246181488037,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35986983.384615384,
|
|
"logits/rejected": -37118111.32576986,
|
|
"logps/chosen": -495.470871040724,
|
|
"logps/rejected": -374.2540012155592,
|
|
"loss": 0.6044,
|
|
"loss/base_kto": 3.758063554763794,
|
|
"metrics/advantage_var": 266.31365966796875,
|
|
"regularization/lipschitz_norm": 1095.349609375,
|
|
"regularization/mmd": 0.1568516045808792,
|
|
"regularization/rkhs_norm": 186.7281036376953,
|
|
"regularization/w1": 0.9200937151908875,
|
|
"rewards/chosen": 0.008832502868560285,
|
|
"rewards/margins": 0.21875871858599344,
|
|
"rewards/rejected": -0.20992621571743314,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 24.467670440673828,
|
|
"kl": 18.701963424682617,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -38700092.23529412,
|
|
"logits/rejected": -38453951.68881685,
|
|
"logps/chosen": -507.2732371794872,
|
|
"logps/rejected": -381.7732728930308,
|
|
"loss": 0.6121,
|
|
"loss/base_kto": 3.743682861328125,
|
|
"metrics/advantage_var": 296.6300354003906,
|
|
"regularization/lipschitz_norm": 1172.72998046875,
|
|
"regularization/mmd": 0.16786277294158936,
|
|
"regularization/rkhs_norm": 199.83665466308594,
|
|
"regularization/w1": 0.9850932359695435,
|
|
"rewards/chosen": 0.34446235300009426,
|
|
"rewards/margins": 0.22968183423489977,
|
|
"rewards/rejected": 0.11478051876519449,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 18.4597225189209,
|
|
"kl": 10.100874900817871,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36295174.440251574,
|
|
"logits/rejected": -37668062.60869565,
|
|
"logps/chosen": -495.23181996855345,
|
|
"logps/rejected": -356.2565751164596,
|
|
"loss": 0.5996,
|
|
"loss/base_kto": 3.7534492015838623,
|
|
"metrics/advantage_var": 248.06846618652344,
|
|
"regularization/lipschitz_norm": 1063.7315673828125,
|
|
"regularization/mmd": 0.14948324859142303,
|
|
"regularization/rkhs_norm": 177.9562530517578,
|
|
"regularization/w1": 0.8935344815254211,
|
|
"rewards/chosen": 0.14588748284105985,
|
|
"rewards/margins": 0.23258878711685688,
|
|
"rewards/rejected": -0.08670130427579702,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 18.682607650756836,
|
|
"kl": 13.347147941589355,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35835687.384615384,
|
|
"logits/rejected": -37218600.585365854,
|
|
"logps/chosen": -465.88516626602564,
|
|
"logps/rejected": -383.6118521341463,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7235100269317627,
|
|
"metrics/advantage_var": 252.84033203125,
|
|
"regularization/lipschitz_norm": 1083.9730224609375,
|
|
"regularization/mmd": 0.15455412864685059,
|
|
"regularization/rkhs_norm": 183.99302673339844,
|
|
"regularization/w1": 0.9105373620986938,
|
|
"rewards/chosen": 0.10605437938983624,
|
|
"rewards/margins": 0.22892328945825516,
|
|
"rewards/rejected": -0.12286891006841892,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 18.112548828125,
|
|
"kl": 8.68852424621582,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37186758.3496063,
|
|
"logits/rejected": -38183454.95813953,
|
|
"logps/chosen": -483.29896653543307,
|
|
"logps/rejected": -385.3201792635659,
|
|
"loss": 0.5967,
|
|
"loss/base_kto": 3.7061691284179688,
|
|
"metrics/advantage_var": 267.4930114746094,
|
|
"regularization/lipschitz_norm": 1083.5208740234375,
|
|
"regularization/mmd": 0.15726128220558167,
|
|
"regularization/rkhs_norm": 187.21580505371094,
|
|
"regularization/w1": 0.9101576209068298,
|
|
"rewards/chosen": 0.17891497274083415,
|
|
"rewards/margins": 0.26187582602178133,
|
|
"rewards/rejected": -0.08296085328094718,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 22.497026443481445,
|
|
"kl": 11.724501609802246,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -37575972.792738274,
|
|
"logits/rejected": -37687391.948303714,
|
|
"logps/chosen": -480.875189107413,
|
|
"logps/rejected": -383.28912560581585,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.8036885261535645,
|
|
"metrics/advantage_var": 244.5902862548828,
|
|
"regularization/lipschitz_norm": 1035.9989013671875,
|
|
"regularization/mmd": 0.15057645738124847,
|
|
"regularization/rkhs_norm": 179.2576904296875,
|
|
"regularization/w1": 0.8702389597892761,
|
|
"rewards/chosen": 0.1268555934057654,
|
|
"rewards/margins": 0.18380138186166076,
|
|
"rewards/rejected": -0.056945788455895346,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 21.271011352539062,
|
|
"kl": 14.250314712524414,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36566237.72561984,
|
|
"logits/rejected": -37276209.3037037,
|
|
"logps/chosen": -463.1956611570248,
|
|
"logps/rejected": -371.7718981481481,
|
|
"loss": 0.5984,
|
|
"loss/base_kto": 3.7788636684417725,
|
|
"metrics/advantage_var": 238.5911102294922,
|
|
"regularization/lipschitz_norm": 1023.1630859375,
|
|
"regularization/mmd": 0.14863084256649017,
|
|
"regularization/rkhs_norm": 176.94146728515625,
|
|
"regularization/w1": 0.8594570159912109,
|
|
"rewards/chosen": 0.18816916725852273,
|
|
"rewards/margins": 0.19544168414491597,
|
|
"rewards/rejected": -0.007272516886393229,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 17.99951934814453,
|
|
"kl": 9.81983470916748,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36518954.47004608,
|
|
"logits/rejected": -37850807.14785374,
|
|
"logps/chosen": -487.53182603686633,
|
|
"logps/rejected": -384.29409280604136,
|
|
"loss": 0.5911,
|
|
"loss/base_kto": 3.7152459621429443,
|
|
"metrics/advantage_var": 247.6333465576172,
|
|
"regularization/lipschitz_norm": 1027.8251953125,
|
|
"regularization/mmd": 0.15019455552101135,
|
|
"regularization/rkhs_norm": 178.8030242919922,
|
|
"regularization/w1": 0.8633732199668884,
|
|
"rewards/chosen": 0.14282357820900538,
|
|
"rewards/margins": 0.2650409475918303,
|
|
"rewards/rejected": -0.12221736938282492,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 25.400270462036133,
|
|
"kl": 10.128746032714844,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36794279.506172836,
|
|
"logits/rejected": -37450661.26582278,
|
|
"logps/chosen": -505.5550733024691,
|
|
"logps/rejected": -378.05038568037975,
|
|
"loss": 0.598,
|
|
"loss/base_kto": 3.719536304473877,
|
|
"metrics/advantage_var": 256.35247802734375,
|
|
"regularization/lipschitz_norm": 1083.0006103515625,
|
|
"regularization/mmd": 0.15447254478931427,
|
|
"regularization/rkhs_norm": 183.89588928222656,
|
|
"regularization/w1": 0.9097205400466919,
|
|
"rewards/chosen": 0.12851585576563707,
|
|
"rewards/margins": 0.24123774559949784,
|
|
"rewards/rejected": -0.11272188983386076,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 20.682348251342773,
|
|
"kl": 14.537857055664062,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35585871.89808917,
|
|
"logits/rejected": -38136034.159509204,
|
|
"logps/chosen": -472.7933917197452,
|
|
"logps/rejected": -360.1613305214724,
|
|
"loss": 0.5861,
|
|
"loss/base_kto": 3.7032296657562256,
|
|
"metrics/advantage_var": 237.9302215576172,
|
|
"regularization/lipschitz_norm": 999.3062744140625,
|
|
"regularization/mmd": 0.14641115069389343,
|
|
"regularization/rkhs_norm": 174.2989959716797,
|
|
"regularization/w1": 0.8394172787666321,
|
|
"rewards/chosen": 0.18850056836559514,
|
|
"rewards/margins": 0.24888291991910838,
|
|
"rewards/rejected": -0.06038235155351323,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 26.200773239135742,
|
|
"kl": 17.288009643554688,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36156413.01457726,
|
|
"logits/rejected": -36837217.400673404,
|
|
"logps/chosen": -480.846027696793,
|
|
"logps/rejected": -401.2182239057239,
|
|
"loss": 0.6014,
|
|
"loss/base_kto": 3.726454496383667,
|
|
"metrics/advantage_var": 262.059326171875,
|
|
"regularization/lipschitz_norm": 1106.60986328125,
|
|
"regularization/mmd": 0.15532676875591278,
|
|
"regularization/rkhs_norm": 184.91282653808594,
|
|
"regularization/w1": 0.9295522570610046,
|
|
"rewards/chosen": 0.2757827714302797,
|
|
"rewards/margins": 0.23156987140339955,
|
|
"rewards/rejected": 0.04421290002688013,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 17.645538330078125,
|
|
"kl": 20.367826461791992,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36389639.08923077,
|
|
"logits/rejected": -37462296.458598725,
|
|
"logps/chosen": -478.2633653846154,
|
|
"logps/rejected": -369.76769008757964,
|
|
"loss": 0.5913,
|
|
"loss/base_kto": 3.510164976119995,
|
|
"metrics/advantage_var": 373.0108642578125,
|
|
"regularization/lipschitz_norm": 1241.9139404296875,
|
|
"regularization/mmd": 0.1766994446516037,
|
|
"regularization/rkhs_norm": 210.3564910888672,
|
|
"regularization/w1": 1.0432077646255493,
|
|
"rewards/chosen": 0.4789816518930288,
|
|
"rewards/margins": 0.5101663616485073,
|
|
"rewards/rejected": -0.03118470975547839,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 14.581066131591797,
|
|
"kl": 12.646075248718262,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35601939.56687898,
|
|
"logits/rejected": -36514652.66257669,
|
|
"logps/chosen": -466.05677746815286,
|
|
"logps/rejected": -359.2162816334356,
|
|
"loss": 0.612,
|
|
"loss/base_kto": 3.278748035430908,
|
|
"metrics/advantage_var": 722.8309936523438,
|
|
"regularization/lipschitz_norm": 1642.021728515625,
|
|
"regularization/mmd": 0.23777084052562714,
|
|
"regularization/rkhs_norm": 283.060546875,
|
|
"regularization/w1": 1.379298210144043,
|
|
"rewards/chosen": 0.49157175440697154,
|
|
"rewards/margins": 0.8237190367323886,
|
|
"rewards/rejected": -0.33214728232541696,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 17.722475051879883,
|
|
"kl": 10.00409984588623,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36037570.234920636,
|
|
"logits/rejected": -37745036.996923074,
|
|
"logps/chosen": -480.87296626984124,
|
|
"logps/rejected": -400.6152403846154,
|
|
"loss": 0.6267,
|
|
"loss/base_kto": 3.3586418628692627,
|
|
"metrics/advantage_var": 968.7614135742188,
|
|
"regularization/lipschitz_norm": 1692.2047119140625,
|
|
"regularization/mmd": 0.23324553668498993,
|
|
"regularization/rkhs_norm": 277.6732482910156,
|
|
"regularization/w1": 1.4214519262313843,
|
|
"rewards/chosen": 0.3753167288643973,
|
|
"rewards/margins": 0.6660401128412603,
|
|
"rewards/rejected": -0.290723383976863,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 24.830984115600586,
|
|
"kl": 11.239001274108887,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36216176.89440994,
|
|
"logits/rejected": -37063876.42767295,
|
|
"logps/chosen": -485.518051242236,
|
|
"logps/rejected": -400.12318199685535,
|
|
"loss": 0.6103,
|
|
"loss/base_kto": 3.2918148040771484,
|
|
"metrics/advantage_var": 631.3637084960938,
|
|
"regularization/lipschitz_norm": 1619.4119873046875,
|
|
"regularization/mmd": 0.22996166348457336,
|
|
"regularization/rkhs_norm": 273.7638854980469,
|
|
"regularization/w1": 1.3603060245513916,
|
|
"rewards/chosen": 0.44590764016098117,
|
|
"rewards/margins": 0.7903715403612073,
|
|
"rewards/rejected": -0.34446390020022605,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 22.797107696533203,
|
|
"kl": 11.124322891235352,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36511150.047543585,
|
|
"logits/rejected": -36602401.9229584,
|
|
"logps/chosen": -507.25445721077654,
|
|
"logps/rejected": -366.174499229584,
|
|
"loss": 0.6072,
|
|
"loss/base_kto": 3.3020102977752686,
|
|
"metrics/advantage_var": 611.6364135742188,
|
|
"regularization/lipschitz_norm": 1584.796875,
|
|
"regularization/mmd": 0.22439594566822052,
|
|
"regularization/rkhs_norm": 267.1380310058594,
|
|
"regularization/w1": 1.331229329109192,
|
|
"rewards/chosen": 0.3487194321234895,
|
|
"rewards/margins": 0.7601920289370969,
|
|
"rewards/rejected": -0.4114725968136075,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 19.19955825805664,
|
|
"kl": 18.63279151916504,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34369725.33744222,
|
|
"logits/rejected": -36344681.07765452,
|
|
"logps/chosen": -486.60472842835134,
|
|
"logps/rejected": -381.2710974643423,
|
|
"loss": 0.6084,
|
|
"loss/base_kto": 3.2762367725372314,
|
|
"metrics/advantage_var": 632.5957641601562,
|
|
"regularization/lipschitz_norm": 1622.939453125,
|
|
"regularization/mmd": 0.227473646402359,
|
|
"regularization/rkhs_norm": 270.8019714355469,
|
|
"regularization/w1": 1.3632692098617554,
|
|
"rewards/chosen": 0.5085174607569337,
|
|
"rewards/margins": 0.7692140653450751,
|
|
"rewards/rejected": -0.26069660458814137,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 23.80451202392578,
|
|
"kl": 15.435348510742188,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36562309.719568565,
|
|
"logits/rejected": -37276362.04120444,
|
|
"logps/chosen": -479.53861710323577,
|
|
"logps/rejected": -380.42415312995246,
|
|
"loss": 0.6057,
|
|
"loss/base_kto": 3.2696053981781006,
|
|
"metrics/advantage_var": 602.6306762695312,
|
|
"regularization/lipschitz_norm": 1605.048095703125,
|
|
"regularization/mmd": 0.22752657532691956,
|
|
"regularization/rkhs_norm": 270.8649597167969,
|
|
"regularization/w1": 1.3482402563095093,
|
|
"rewards/chosen": 0.5724000151976598,
|
|
"rewards/margins": 0.7972763101876311,
|
|
"rewards/rejected": -0.22487629498997128,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 48.86388397216797,
|
|
"kl": 14.562422752380371,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34969252.827371694,
|
|
"logits/rejected": -37211677.73940346,
|
|
"logps/chosen": -468.6008456454121,
|
|
"logps/rejected": -369.25431711145995,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.3056893348693848,
|
|
"metrics/advantage_var": 624.78662109375,
|
|
"regularization/lipschitz_norm": 1547.953125,
|
|
"regularization/mmd": 0.2235708236694336,
|
|
"regularization/rkhs_norm": 266.1557312011719,
|
|
"regularization/w1": 1.3002806901931763,
|
|
"rewards/chosen": 0.478918191248299,
|
|
"rewards/margins": 0.7449161572541123,
|
|
"rewards/rejected": -0.2659979660058134,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 21.663223266601562,
|
|
"kl": 21.121829986572266,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36329721.294298925,
|
|
"logits/rejected": -37947297.87638669,
|
|
"logps/chosen": -514.2784572419106,
|
|
"logps/rejected": -378.4793977812995,
|
|
"loss": 0.6118,
|
|
"loss/base_kto": 3.255121946334839,
|
|
"metrics/advantage_var": 602.9996337890625,
|
|
"regularization/lipschitz_norm": 1677.8475341796875,
|
|
"regularization/mmd": 0.22991926968097687,
|
|
"regularization/rkhs_norm": 273.7134094238281,
|
|
"regularization/w1": 1.4093917608261108,
|
|
"rewards/chosen": 0.6202737722999326,
|
|
"rewards/margins": 0.8242826213335845,
|
|
"rewards/rejected": -0.20400884903365193,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 19.38195037841797,
|
|
"kl": 13.08390998840332,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34997597.69678407,
|
|
"logits/rejected": -35992002.755980864,
|
|
"logps/chosen": -506.90438361408883,
|
|
"logps/rejected": -375.1045155502392,
|
|
"loss": 0.6042,
|
|
"loss/base_kto": 3.2489612102508545,
|
|
"metrics/advantage_var": 650.0698852539062,
|
|
"regularization/lipschitz_norm": 1610.8154296875,
|
|
"regularization/mmd": 0.23186349868774414,
|
|
"regularization/rkhs_norm": 276.0279846191406,
|
|
"regularization/w1": 1.353084921836853,
|
|
"rewards/chosen": 0.5146272200720233,
|
|
"rewards/margins": 0.853607911546804,
|
|
"rewards/rejected": -0.3389806914747807,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 19.10856819152832,
|
|
"kl": 15.936148643493652,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -36025036.8,
|
|
"logits/rejected": -37407968.0,
|
|
"logps/chosen": -461.17919921875,
|
|
"logps/rejected": -373.235205078125,
|
|
"loss": 0.6105,
|
|
"loss/base_kto": 3.282215118408203,
|
|
"metrics/advantage_var": 697.1913452148438,
|
|
"regularization/lipschitz_norm": 1630.668212890625,
|
|
"regularization/mmd": 0.23203685879707336,
|
|
"regularization/rkhs_norm": 276.2343444824219,
|
|
"regularization/w1": 1.369761347770691,
|
|
"rewards/chosen": 0.5281445503234863,
|
|
"rewards/margins": 0.783744716644287,
|
|
"rewards/rejected": -0.2556001663208008,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 15.50611400604248,
|
|
"kl": 12.658308982849121,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35978363.01298701,
|
|
"logits/rejected": -37760573.686746985,
|
|
"logps/chosen": -470.4105113636364,
|
|
"logps/rejected": -368.535203313253,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.2967498302459717,
|
|
"metrics/advantage_var": 518.6314697265625,
|
|
"regularization/lipschitz_norm": 1476.4432373046875,
|
|
"regularization/mmd": 0.21278348565101624,
|
|
"regularization/rkhs_norm": 253.3136749267578,
|
|
"regularization/w1": 1.240212321281433,
|
|
"rewards/chosen": 0.3881315751509233,
|
|
"rewards/margins": 0.7245794632609808,
|
|
"rewards/rejected": -0.33644788811005744,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 16.433752059936523,
|
|
"kl": 9.784865379333496,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -33988953.72347267,
|
|
"logits/rejected": -36039717.34954407,
|
|
"logps/chosen": -477.02783360128615,
|
|
"logps/rejected": -367.58572378419456,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.2700486183166504,
|
|
"metrics/advantage_var": 608.6986694335938,
|
|
"regularization/lipschitz_norm": 1575.5340576171875,
|
|
"regularization/mmd": 0.22585450112819672,
|
|
"regularization/rkhs_norm": 268.8744201660156,
|
|
"regularization/w1": 1.323448657989502,
|
|
"rewards/chosen": 0.4100186755802854,
|
|
"rewards/margins": 0.8063638199275023,
|
|
"rewards/rejected": -0.39634514434721696,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 19.169435501098633,
|
|
"kl": 10.167763710021973,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -36013923.93157076,
|
|
"logits/rejected": -36364095.09576138,
|
|
"logps/chosen": -492.5692068429238,
|
|
"logps/rejected": -398.8990875196232,
|
|
"loss": 0.6232,
|
|
"loss/base_kto": 3.2810959815979004,
|
|
"metrics/advantage_var": 733.5505981445312,
|
|
"regularization/lipschitz_norm": 1737.5484619140625,
|
|
"regularization/mmd": 0.24507227540016174,
|
|
"regularization/rkhs_norm": 291.752685546875,
|
|
"regularization/w1": 1.4595407247543335,
|
|
"rewards/chosen": 0.4657984803290241,
|
|
"rewards/margins": 0.8314784982045638,
|
|
"rewards/rejected": -0.36568001787553966,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 15.603521347045898,
|
|
"kl": 12.8410062789917,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34583273.01547988,
|
|
"logits/rejected": -36524241.96845426,
|
|
"logps/chosen": -488.2435178018576,
|
|
"logps/rejected": -379.0256555599369,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.2975711822509766,
|
|
"metrics/advantage_var": 570.064697265625,
|
|
"regularization/lipschitz_norm": 1525.0550537109375,
|
|
"regularization/mmd": 0.21888184547424316,
|
|
"regularization/rkhs_norm": 260.57366943359375,
|
|
"regularization/w1": 1.2810462713241577,
|
|
"rewards/chosen": 0.39716582401618133,
|
|
"rewards/margins": 0.7907970942713815,
|
|
"rewards/rejected": -0.3936312702552001,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 22.659696578979492,
|
|
"kl": 13.170989036560059,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35158830.72698908,
|
|
"logits/rejected": -35037594.24100156,
|
|
"logps/chosen": -474.0039976599064,
|
|
"logps/rejected": -377.1842478482003,
|
|
"loss": 0.6033,
|
|
"loss/base_kto": 3.3246707916259766,
|
|
"metrics/advantage_var": 544.271484375,
|
|
"regularization/lipschitz_norm": 1528.093994140625,
|
|
"regularization/mmd": 0.2184295952320099,
|
|
"regularization/rkhs_norm": 260.0352478027344,
|
|
"regularization/w1": 1.2835988998413086,
|
|
"rewards/chosen": 0.48960388469249705,
|
|
"rewards/margins": 0.7395275191096772,
|
|
"rewards/rejected": -0.24992363441718016,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 21.347990036010742,
|
|
"kl": 8.728641510009766,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -33387603.11179173,
|
|
"logits/rejected": -35701879.22169059,
|
|
"logps/chosen": -460.062547856049,
|
|
"logps/rejected": -376.56215111642746,
|
|
"loss": 0.5983,
|
|
"loss/base_kto": 3.331209659576416,
|
|
"metrics/advantage_var": 495.9743347167969,
|
|
"regularization/lipschitz_norm": 1483.7119140625,
|
|
"regularization/mmd": 0.20908823609352112,
|
|
"regularization/rkhs_norm": 248.91455078125,
|
|
"regularization/w1": 1.2463181018829346,
|
|
"rewards/chosen": 0.3678303787208078,
|
|
"rewards/margins": 0.7245289335796345,
|
|
"rewards/rejected": -0.35669855485882673,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 19.782438278198242,
|
|
"kl": 17.806982040405273,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35410609.168253966,
|
|
"logits/rejected": -36458754.363076925,
|
|
"logps/chosen": -471.03184523809523,
|
|
"logps/rejected": -367.13396634615384,
|
|
"loss": 0.6164,
|
|
"loss/base_kto": 3.304821729660034,
|
|
"metrics/advantage_var": 631.5667724609375,
|
|
"regularization/lipschitz_norm": 1657.6304931640625,
|
|
"regularization/mmd": 0.23422546684741974,
|
|
"regularization/rkhs_norm": 278.83984375,
|
|
"regularization/w1": 1.3924096822738647,
|
|
"rewards/chosen": 0.54564451187376,
|
|
"rewards/margins": 0.7249720687773056,
|
|
"rewards/rejected": -0.17932755690354568,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 16.90350341796875,
|
|
"kl": 15.593917846679688,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34073868.15313936,
|
|
"logits/rejected": -34781618.42424242,
|
|
"logps/chosen": -470.34126148545175,
|
|
"logps/rejected": -373.5643191786284,
|
|
"loss": 0.6019,
|
|
"loss/base_kto": 3.309987783432007,
|
|
"metrics/advantage_var": 547.5170288085938,
|
|
"regularization/lipschitz_norm": 1533.3909912109375,
|
|
"regularization/mmd": 0.21713657677173615,
|
|
"regularization/rkhs_norm": 258.4959411621094,
|
|
"regularization/w1": 1.288048505783081,
|
|
"rewards/chosen": 0.4688326264596095,
|
|
"rewards/margins": 0.7340357099907473,
|
|
"rewards/rejected": -0.26520308353113786,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 17.07435417175293,
|
|
"kl": 16.390722274780273,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35766201.1572327,
|
|
"logits/rejected": -36289828.571428575,
|
|
"logps/chosen": -485.7150157232704,
|
|
"logps/rejected": -376.8792216614907,
|
|
"loss": 0.6122,
|
|
"loss/base_kto": 3.317823886871338,
|
|
"metrics/advantage_var": 673.7321166992188,
|
|
"regularization/lipschitz_norm": 1607.3251953125,
|
|
"regularization/mmd": 0.22924481332302094,
|
|
"regularization/rkhs_norm": 272.91046142578125,
|
|
"regularization/w1": 1.3501532077789307,
|
|
"rewards/chosen": 0.54358586725199,
|
|
"rewards/margins": 0.7645848089933647,
|
|
"rewards/rejected": -0.22099894174137472,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 20.586442947387695,
|
|
"kl": 8.326224327087402,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35506605.41935484,
|
|
"logits/rejected": -36913164.412121214,
|
|
"logps/chosen": -498.81612903225806,
|
|
"logps/rejected": -383.63522727272726,
|
|
"loss": 0.6178,
|
|
"loss/base_kto": 3.3446462154388428,
|
|
"metrics/advantage_var": 600.1603393554688,
|
|
"regularization/lipschitz_norm": 1631.9766845703125,
|
|
"regularization/mmd": 0.226824089884758,
|
|
"regularization/rkhs_norm": 270.0286865234375,
|
|
"regularization/w1": 1.3708603382110596,
|
|
"rewards/chosen": 0.33258100940335183,
|
|
"rewards/margins": 0.706335591663946,
|
|
"rewards/rejected": -0.3737545822605942,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 14.240646362304688,
|
|
"kl": 11.048199653625488,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33645490.20775194,
|
|
"logits/rejected": -35844999.055118114,
|
|
"logps/chosen": -481.35532945736435,
|
|
"logps/rejected": -356.21001476377955,
|
|
"loss": 0.5999,
|
|
"loss/base_kto": 3.2740700244903564,
|
|
"metrics/advantage_var": 554.83984375,
|
|
"regularization/lipschitz_norm": 1556.0499267578125,
|
|
"regularization/mmd": 0.21828582882881165,
|
|
"regularization/rkhs_norm": 259.8641052246094,
|
|
"regularization/w1": 1.3070820569992065,
|
|
"rewards/chosen": 0.45705481240915696,
|
|
"rewards/margins": 0.8093554052671786,
|
|
"rewards/rejected": -0.3523005928580217,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 21.553014755249023,
|
|
"kl": 11.094316482543945,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34676400.98765432,
|
|
"logits/rejected": -36567422.37974683,
|
|
"logps/chosen": -501.79306520061726,
|
|
"logps/rejected": -371.48116099683546,
|
|
"loss": 0.6124,
|
|
"loss/base_kto": 3.364614963531494,
|
|
"metrics/advantage_var": 587.3946533203125,
|
|
"regularization/lipschitz_norm": 1561.5902099609375,
|
|
"regularization/mmd": 0.2229761928319931,
|
|
"regularization/rkhs_norm": 265.4478454589844,
|
|
"regularization/w1": 1.3117358684539795,
|
|
"rewards/chosen": 0.3346147890444155,
|
|
"rewards/margins": 0.7078894835595955,
|
|
"rewards/rejected": -0.37327469451517997,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 18.022768020629883,
|
|
"kl": 15.335441589355469,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33522519.174114022,
|
|
"logits/rejected": -34918646.668779716,
|
|
"logps/chosen": -460.5504622496148,
|
|
"logps/rejected": -354.3644512678288,
|
|
"loss": 0.6058,
|
|
"loss/base_kto": 3.3383285999298096,
|
|
"metrics/advantage_var": 562.71142578125,
|
|
"regularization/lipschitz_norm": 1535.8997802734375,
|
|
"regularization/mmd": 0.21826408803462982,
|
|
"regularization/rkhs_norm": 259.83819580078125,
|
|
"regularization/w1": 1.2901560068130493,
|
|
"rewards/chosen": 0.524051143134871,
|
|
"rewards/margins": 0.7177906820379638,
|
|
"rewards/rejected": -0.1937395389030928,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 17.134130477905273,
|
|
"kl": 6.6206889152526855,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35148567.578947365,
|
|
"logits/rejected": -37054936.38095238,
|
|
"logps/chosen": -505.05653782894734,
|
|
"logps/rejected": -380.48214285714283,
|
|
"loss": 0.6256,
|
|
"loss/base_kto": 3.3530690670013428,
|
|
"metrics/advantage_var": 691.9746704101562,
|
|
"regularization/lipschitz_norm": 1683.4412841796875,
|
|
"regularization/mmd": 0.23729148507118225,
|
|
"regularization/rkhs_norm": 282.4898376464844,
|
|
"regularization/w1": 1.4140905141830444,
|
|
"rewards/chosen": 0.25868932824385793,
|
|
"rewards/margins": 0.7247344354041536,
|
|
"rewards/rejected": -0.46604510716029574,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 18.29180145263672,
|
|
"kl": 10.3164701461792,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -33816487.64252696,
|
|
"logits/rejected": -34764576.05071315,
|
|
"logps/chosen": -490.7649268104777,
|
|
"logps/rejected": -366.11660558637084,
|
|
"loss": 0.5993,
|
|
"loss/base_kto": 3.299171209335327,
|
|
"metrics/advantage_var": 547.89013671875,
|
|
"regularization/lipschitz_norm": 1523.284423828125,
|
|
"regularization/mmd": 0.21546481549739838,
|
|
"regularization/rkhs_norm": 256.5057067871094,
|
|
"regularization/w1": 1.2795588970184326,
|
|
"rewards/chosen": 0.41266320556264446,
|
|
"rewards/margins": 0.7335773032715143,
|
|
"rewards/rejected": -0.32091409770886986,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 16.146881103515625,
|
|
"kl": 13.290518760681152,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33954004.08037094,
|
|
"logits/rejected": -34616496.328594,
|
|
"logps/chosen": -471.5366112828439,
|
|
"logps/rejected": -361.43907977883094,
|
|
"loss": 0.5996,
|
|
"loss/base_kto": 3.35132098197937,
|
|
"metrics/advantage_var": 550.5408325195312,
|
|
"regularization/lipschitz_norm": 1469.648681640625,
|
|
"regularization/mmd": 0.2109934389591217,
|
|
"regularization/rkhs_norm": 251.18264770507812,
|
|
"regularization/w1": 1.2345049381256104,
|
|
"rewards/chosen": 0.42645782517870945,
|
|
"rewards/margins": 0.6939193759229334,
|
|
"rewards/rejected": -0.26746155074422395,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 20.550386428833008,
|
|
"kl": 12.950957298278809,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34710448.53879941,
|
|
"logits/rejected": -34624297.59463987,
|
|
"logps/chosen": -472.0133144216691,
|
|
"logps/rejected": -361.9915201005025,
|
|
"loss": 0.5968,
|
|
"loss/base_kto": 3.3124001026153564,
|
|
"metrics/advantage_var": 535.0121459960938,
|
|
"regularization/lipschitz_norm": 1486.1500244140625,
|
|
"regularization/mmd": 0.21363840997219086,
|
|
"regularization/rkhs_norm": 254.33140563964844,
|
|
"regularization/w1": 1.2483659982681274,
|
|
"rewards/chosen": 0.5021344432020955,
|
|
"rewards/margins": 0.7558174599296134,
|
|
"rewards/rejected": -0.2536830167275178,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 14.962956428527832,
|
|
"kl": 16.535037994384766,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35013050.64231355,
|
|
"logits/rejected": -37614593.64365971,
|
|
"logps/chosen": -467.08485540334857,
|
|
"logps/rejected": -364.5472010433387,
|
|
"loss": 0.6021,
|
|
"loss/base_kto": 3.331411838531494,
|
|
"metrics/advantage_var": 533.7169799804688,
|
|
"regularization/lipschitz_norm": 1511.59228515625,
|
|
"regularization/mmd": 0.2157890796661377,
|
|
"regularization/rkhs_norm": 256.8918151855469,
|
|
"regularization/w1": 1.2697376012802124,
|
|
"rewards/chosen": 0.480613766558457,
|
|
"rewards/margins": 0.7152077779417586,
|
|
"rewards/rejected": -0.23459401138330158,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 19.377729415893555,
|
|
"kl": 22.779943466186523,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34853013.77910448,
|
|
"logits/rejected": -35985589.29836065,
|
|
"logps/chosen": -465.0221082089552,
|
|
"logps/rejected": -394.65512295081965,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.3015334606170654,
|
|
"metrics/advantage_var": 565.37890625,
|
|
"regularization/lipschitz_norm": 1508.641845703125,
|
|
"regularization/mmd": 0.21362176537513733,
|
|
"regularization/rkhs_norm": 254.31163024902344,
|
|
"regularization/w1": 1.267259120941162,
|
|
"rewards/chosen": 0.618648289922458,
|
|
"rewards/margins": 0.7440603022415538,
|
|
"rewards/rejected": -0.1254120123190958,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 19.16179084777832,
|
|
"kl": 18.917919158935547,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34662096.7133758,
|
|
"logits/rejected": -35347914.60122699,
|
|
"logps/chosen": -455.1887440286624,
|
|
"logps/rejected": -390.7230157208589,
|
|
"loss": 0.6095,
|
|
"loss/base_kto": 3.3978583812713623,
|
|
"metrics/advantage_var": 583.5626831054688,
|
|
"regularization/lipschitz_norm": 1502.7821044921875,
|
|
"regularization/mmd": 0.21602332592010498,
|
|
"regularization/rkhs_norm": 257.1706237792969,
|
|
"regularization/w1": 1.2623369693756104,
|
|
"rewards/chosen": 0.507355125087082,
|
|
"rewards/margins": 0.6383283442272577,
|
|
"rewards/rejected": -0.13097321914017565,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 27.668109893798828,
|
|
"kl": 20.363744735717773,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33695011.45814308,
|
|
"logits/rejected": -34444677.547351524,
|
|
"logps/chosen": -499.056697108067,
|
|
"logps/rejected": -376.24887138844304,
|
|
"loss": 0.6014,
|
|
"loss/base_kto": 3.3564345836639404,
|
|
"metrics/advantage_var": 530.4850463867188,
|
|
"regularization/lipschitz_norm": 1482.207275390625,
|
|
"regularization/mmd": 0.20999467372894287,
|
|
"regularization/rkhs_norm": 249.99365234375,
|
|
"regularization/w1": 1.2450541257858276,
|
|
"rewards/chosen": 0.5535609508032486,
|
|
"rewards/margins": 0.6984056427824258,
|
|
"rewards/rejected": -0.14484469197917713,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 20.353731155395508,
|
|
"kl": 17.986051559448242,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -32922226.602254428,
|
|
"logits/rejected": -34518328.32776935,
|
|
"logps/chosen": -479.0992351046699,
|
|
"logps/rejected": -366.3035375569044,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.274026870727539,
|
|
"metrics/advantage_var": 492.223388671875,
|
|
"regularization/lipschitz_norm": 1502.2119140625,
|
|
"regularization/mmd": 0.20957601070404053,
|
|
"regularization/rkhs_norm": 249.49526977539062,
|
|
"regularization/w1": 1.2618581056594849,
|
|
"rewards/chosen": 0.5238556547057418,
|
|
"rewards/margins": 0.763230188838009,
|
|
"rewards/rejected": -0.23937453413226717,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 33.46779251098633,
|
|
"kl": 13.584640502929688,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34771941.01085272,
|
|
"logits/rejected": -37004770.16692913,
|
|
"logps/chosen": -504.6795058139535,
|
|
"logps/rejected": -387.7888533464567,
|
|
"loss": 0.6056,
|
|
"loss/base_kto": 3.319662570953369,
|
|
"metrics/advantage_var": 564.02587890625,
|
|
"regularization/lipschitz_norm": 1554.7366943359375,
|
|
"regularization/mmd": 0.21924880146980286,
|
|
"regularization/rkhs_norm": 261.010498046875,
|
|
"regularization/w1": 1.3059788942337036,
|
|
"rewards/chosen": 0.42508180603500484,
|
|
"rewards/margins": 0.7293694536728001,
|
|
"rewards/rejected": -0.30428764763779526,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 17.88528060913086,
|
|
"kl": 18.6299991607666,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -32324284.369529985,
|
|
"logits/rejected": -34988773.357466064,
|
|
"logps/chosen": -468.8882192058347,
|
|
"logps/rejected": -365.79965592006033,
|
|
"loss": 0.5903,
|
|
"loss/base_kto": 3.2499802112579346,
|
|
"metrics/advantage_var": 545.6959838867188,
|
|
"regularization/lipschitz_norm": 1498.6328125,
|
|
"regularization/mmd": 0.21394610404968262,
|
|
"regularization/rkhs_norm": 254.69775390625,
|
|
"regularization/w1": 1.2588516473770142,
|
|
"rewards/chosen": 0.5019449144360312,
|
|
"rewards/margins": 0.7761924190831725,
|
|
"rewards/rejected": -0.2742475046471413,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 18.10062026977539,
|
|
"kl": 9.969759941101074,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -33411004.27086614,
|
|
"logits/rejected": -35512853.43255814,
|
|
"logps/chosen": -455.75275590551183,
|
|
"logps/rejected": -363.99176356589146,
|
|
"loss": 0.6041,
|
|
"loss/base_kto": 3.375481367111206,
|
|
"metrics/advantage_var": 507.2935485839844,
|
|
"regularization/lipschitz_norm": 1484.4630126953125,
|
|
"regularization/mmd": 0.2105344831943512,
|
|
"regularization/rkhs_norm": 250.63633728027344,
|
|
"regularization/w1": 1.2469488382339478,
|
|
"rewards/chosen": 0.3615355243833046,
|
|
"rewards/margins": 0.6633751192612721,
|
|
"rewards/rejected": -0.3018395948779675,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 15.774064064025879,
|
|
"kl": 8.933207511901855,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35389627.356466874,
|
|
"logits/rejected": -36346011.343653254,
|
|
"logps/chosen": -511.0354889589905,
|
|
"logps/rejected": -374.84123452012386,
|
|
"loss": 0.6025,
|
|
"loss/base_kto": 3.268543243408203,
|
|
"metrics/advantage_var": 586.6609497070312,
|
|
"regularization/lipschitz_norm": 1580.751953125,
|
|
"regularization/mmd": 0.22369682788848877,
|
|
"regularization/rkhs_norm": 266.3057556152344,
|
|
"regularization/w1": 1.3278316259384155,
|
|
"rewards/chosen": 0.3532002927377021,
|
|
"rewards/margins": 0.8226004210815971,
|
|
"rewards/rejected": -0.4694001283438951,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 15.750038146972656,
|
|
"kl": 10.857072830200195,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34974265.25239617,
|
|
"logits/rejected": -36301883.49847095,
|
|
"logps/chosen": -494.6328873801917,
|
|
"logps/rejected": -365.9459097859327,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.2659287452697754,
|
|
"metrics/advantage_var": 526.2587890625,
|
|
"regularization/lipschitz_norm": 1526.7413330078125,
|
|
"regularization/mmd": 0.2148735374212265,
|
|
"regularization/rkhs_norm": 255.8018341064453,
|
|
"regularization/w1": 1.2824627161026,
|
|
"rewards/chosen": 0.41584529206394766,
|
|
"rewards/margins": 0.7909994664953315,
|
|
"rewards/rejected": -0.37515417443138377,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 17.60833168029785,
|
|
"kl": 8.485814094543457,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35663408.801292405,
|
|
"logits/rejected": -35159800.64145234,
|
|
"logps/chosen": -486.67154684975765,
|
|
"logps/rejected": -404.29850605143724,
|
|
"loss": 0.6112,
|
|
"loss/base_kto": 3.3222947120666504,
|
|
"metrics/advantage_var": 591.0783081054688,
|
|
"regularization/lipschitz_norm": 1599.240234375,
|
|
"regularization/mmd": 0.22431206703186035,
|
|
"regularization/rkhs_norm": 267.0381774902344,
|
|
"regularization/w1": 1.343361735343933,
|
|
"rewards/chosen": 0.26061789862366214,
|
|
"rewards/margins": 0.7497240927778792,
|
|
"rewards/rejected": -0.4891061941542171,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 17.947343826293945,
|
|
"kl": 6.821866512298584,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34982428.57315234,
|
|
"logits/rejected": -35088347.36910569,
|
|
"logps/chosen": -474.1717100301659,
|
|
"logps/rejected": -404.2780741869919,
|
|
"loss": 0.5755,
|
|
"loss/base_kto": 3.2221360206604004,
|
|
"metrics/advantage_var": 465.04351806640625,
|
|
"regularization/lipschitz_norm": 1403.3729248046875,
|
|
"regularization/mmd": 0.20289479196071625,
|
|
"regularization/rkhs_norm": 241.54141235351562,
|
|
"regularization/w1": 1.178833246231079,
|
|
"rewards/chosen": 0.4334717717465592,
|
|
"rewards/margins": 0.8401581979247859,
|
|
"rewards/rejected": -0.40668642617822665,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 16.408185958862305,
|
|
"kl": 9.983183860778809,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34488626.06041335,
|
|
"logits/rejected": -35019982.05837174,
|
|
"logps/chosen": -476.0058624801272,
|
|
"logps/rejected": -371.22652649769583,
|
|
"loss": 0.5656,
|
|
"loss/base_kto": 3.2505149841308594,
|
|
"metrics/advantage_var": 410.3097839355469,
|
|
"regularization/lipschitz_norm": 1294.3602294921875,
|
|
"regularization/mmd": 0.18716847896575928,
|
|
"regularization/rkhs_norm": 222.8196258544922,
|
|
"regularization/w1": 1.087262511253357,
|
|
"rewards/chosen": 0.41420029001887915,
|
|
"rewards/margins": 0.7896021282365251,
|
|
"rewards/rejected": -0.37540183821764594,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 16.845909118652344,
|
|
"kl": 8.303881645202637,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33243454.13592233,
|
|
"logits/rejected": -35099493.31722055,
|
|
"logps/chosen": -481.70378236245955,
|
|
"logps/rejected": -366.5403606495468,
|
|
"loss": 0.5581,
|
|
"loss/base_kto": 3.1919381618499756,
|
|
"metrics/advantage_var": 431.11602783203125,
|
|
"regularization/lipschitz_norm": 1283.7459716796875,
|
|
"regularization/mmd": 0.1944064199924469,
|
|
"regularization/rkhs_norm": 231.4362335205078,
|
|
"regularization/w1": 1.078346610069275,
|
|
"rewards/chosen": 0.4814342511124595,
|
|
"rewards/margins": 0.8682639154522915,
|
|
"rewards/rejected": -0.38682966433983196,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 18.615808486938477,
|
|
"kl": 8.465904235839844,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -33294290.051282052,
|
|
"logits/rejected": -35143998.43902439,
|
|
"logps/chosen": -470.43569711538464,
|
|
"logps/rejected": -368.7186309070122,
|
|
"loss": 0.5498,
|
|
"loss/base_kto": 3.2166802883148193,
|
|
"metrics/advantage_var": 385.70819091796875,
|
|
"regularization/lipschitz_norm": 1189.2044677734375,
|
|
"regularization/mmd": 0.18294848501682281,
|
|
"regularization/rkhs_norm": 217.7958221435547,
|
|
"regularization/w1": 0.9989317059516907,
|
|
"rewards/chosen": 0.3834571104783278,
|
|
"rewards/margins": 0.8170791450629314,
|
|
"rewards/rejected": -0.43362203458460363,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 14.12332534790039,
|
|
"kl": 4.959784984588623,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33501098.13664596,
|
|
"logits/rejected": -34493008.50314465,
|
|
"logps/chosen": -511.83021156832297,
|
|
"logps/rejected": -377.47845420597486,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.2280776500701904,
|
|
"metrics/advantage_var": 395.3878479003906,
|
|
"regularization/lipschitz_norm": 1310.1051025390625,
|
|
"regularization/mmd": 0.1888968050479889,
|
|
"regularization/rkhs_norm": 224.8771514892578,
|
|
"regularization/w1": 1.1004881858825684,
|
|
"rewards/chosen": 0.3688108313898122,
|
|
"rewards/margins": 0.8398954740999881,
|
|
"rewards/rejected": -0.4710846427101759,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 17.490766525268555,
|
|
"kl": 8.799834251403809,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -32949561.532428358,
|
|
"logits/rejected": -34604123.28038898,
|
|
"logps/chosen": -469.9966063348416,
|
|
"logps/rejected": -370.9157718800648,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.2049686908721924,
|
|
"metrics/advantage_var": 434.2018737792969,
|
|
"regularization/lipschitz_norm": 1275.1317138671875,
|
|
"regularization/mmd": 0.19241748750209808,
|
|
"regularization/rkhs_norm": 229.0684356689453,
|
|
"regularization/w1": 1.0711106061935425,
|
|
"rewards/chosen": 0.45355602051517724,
|
|
"rewards/margins": 0.8488865145101884,
|
|
"rewards/rejected": -0.39533049399501113,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.122283935546875,
|
|
"kl": 12.252435684204102,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33630768.87636933,
|
|
"logits/rejected": -35612505.85959438,
|
|
"logps/chosen": -494.43446791862283,
|
|
"logps/rejected": -406.71265600624025,
|
|
"loss": 0.5624,
|
|
"loss/base_kto": 3.14216685295105,
|
|
"metrics/advantage_var": 451.7399597167969,
|
|
"regularization/lipschitz_norm": 1377.656494140625,
|
|
"regularization/mmd": 0.19991855323314667,
|
|
"regularization/rkhs_norm": 237.998291015625,
|
|
"regularization/w1": 1.1572314500808716,
|
|
"rewards/chosen": 0.5083081360340864,
|
|
"rewards/margins": 0.914226203811485,
|
|
"rewards/rejected": -0.4059180677773986,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.93985366821289,
|
|
"kl": 9.2858304977417,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34496537.40155039,
|
|
"logits/rejected": -35233724.27086614,
|
|
"logps/chosen": -457.42427325581394,
|
|
"logps/rejected": -402.57536909448817,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.255056381225586,
|
|
"metrics/advantage_var": 400.959228515625,
|
|
"regularization/lipschitz_norm": 1238.921875,
|
|
"regularization/mmd": 0.1878456175327301,
|
|
"regularization/rkhs_norm": 223.625732421875,
|
|
"regularization/w1": 1.040694236755371,
|
|
"rewards/chosen": 0.39651699805444524,
|
|
"rewards/margins": 0.815335049850705,
|
|
"rewards/rejected": -0.41881805179625986,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 14.545886993408203,
|
|
"kl": 13.314804077148438,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33729181.786163524,
|
|
"logits/rejected": -35633031.1552795,
|
|
"logps/chosen": -485.95700668238993,
|
|
"logps/rejected": -368.83933423913044,
|
|
"loss": 0.5529,
|
|
"loss/base_kto": 3.202782154083252,
|
|
"metrics/advantage_var": 382.2320251464844,
|
|
"regularization/lipschitz_norm": 1233.8470458984375,
|
|
"regularization/mmd": 0.18430650234222412,
|
|
"regularization/rkhs_norm": 219.41250610351562,
|
|
"regularization/w1": 1.0364315509796143,
|
|
"rewards/chosen": 0.4813539517000786,
|
|
"rewards/margins": 0.8171038918022234,
|
|
"rewards/rejected": -0.3357499401021448,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 17.223203659057617,
|
|
"kl": 11.675740242004395,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34248610.46929134,
|
|
"logits/rejected": -34210255.578294575,
|
|
"logps/chosen": -477.9596456692913,
|
|
"logps/rejected": -377.89670542635656,
|
|
"loss": 0.5615,
|
|
"loss/base_kto": 3.249102830886841,
|
|
"metrics/advantage_var": 388.05426025390625,
|
|
"regularization/lipschitz_norm": 1258.7603759765625,
|
|
"regularization/mmd": 0.18586070835590363,
|
|
"regularization/rkhs_norm": 221.2627410888672,
|
|
"regularization/w1": 1.0573586225509644,
|
|
"rewards/chosen": 0.4373135303887795,
|
|
"rewards/margins": 0.7716517673528056,
|
|
"rewards/rejected": -0.33433823696402615,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 14.629900932312012,
|
|
"kl": 10.75137710571289,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35498980.75743349,
|
|
"logits/rejected": -35540998.3900156,
|
|
"logps/chosen": -479.5279244913928,
|
|
"logps/rejected": -372.66873049922,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.206221103668213,
|
|
"metrics/advantage_var": 418.78106689453125,
|
|
"regularization/lipschitz_norm": 1310.5302734375,
|
|
"regularization/mmd": 0.19171538949012756,
|
|
"regularization/rkhs_norm": 228.2325897216797,
|
|
"regularization/w1": 1.1008453369140625,
|
|
"rewards/chosen": 0.4715915733659771,
|
|
"rewards/margins": 0.834670483729423,
|
|
"rewards/rejected": -0.3630789103634458,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 14.324227333068848,
|
|
"kl": 10.652953147888184,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -33992343.502276175,
|
|
"logits/rejected": -35529716.45732689,
|
|
"logps/chosen": -469.60124241274656,
|
|
"logps/rejected": -362.1941425120773,
|
|
"loss": 0.5576,
|
|
"loss/base_kto": 3.2229363918304443,
|
|
"metrics/advantage_var": 393.4222717285156,
|
|
"regularization/lipschitz_norm": 1256.2760009765625,
|
|
"regularization/mmd": 0.18269923329353333,
|
|
"regularization/rkhs_norm": 217.4990692138672,
|
|
"regularization/w1": 1.0552717447280884,
|
|
"rewards/chosen": 0.46617642809299126,
|
|
"rewards/margins": 0.8198295296224674,
|
|
"rewards/rejected": -0.35365310152947615,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 19.66016960144043,
|
|
"kl": 13.546658515930176,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35092717.44927536,
|
|
"logits/rejected": -37277835.848254934,
|
|
"logps/chosen": -495.62847222222223,
|
|
"logps/rejected": -386.6493740515933,
|
|
"loss": 0.5729,
|
|
"loss/base_kto": 3.199143171310425,
|
|
"metrics/advantage_var": 489.4619140625,
|
|
"regularization/lipschitz_norm": 1405.9876708984375,
|
|
"regularization/mmd": 0.2029687464237213,
|
|
"regularization/rkhs_norm": 241.6294708251953,
|
|
"regularization/w1": 1.1810296773910522,
|
|
"rewards/chosen": 0.5008374888347926,
|
|
"rewards/margins": 0.8418600092329456,
|
|
"rewards/rejected": -0.34102252039815295,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 19.954313278198242,
|
|
"kl": 10.985320091247559,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34574593.61006289,
|
|
"logits/rejected": -36158731.13043478,
|
|
"logps/chosen": -475.04795597484275,
|
|
"logps/rejected": -372.2355638586956,
|
|
"loss": 0.552,
|
|
"loss/base_kto": 3.1939830780029297,
|
|
"metrics/advantage_var": 377.78436279296875,
|
|
"regularization/lipschitz_norm": 1235.314208984375,
|
|
"regularization/mmd": 0.18432655930519104,
|
|
"regularization/rkhs_norm": 219.43638610839844,
|
|
"regularization/w1": 1.0376638174057007,
|
|
"rewards/chosen": 0.4707678069108687,
|
|
"rewards/margins": 0.836925907299674,
|
|
"rewards/rejected": -0.3661581003888053,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 21.724349975585938,
|
|
"kl": 12.762263298034668,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35928067.21507064,
|
|
"logits/rejected": -36228107.147744946,
|
|
"logps/chosen": -489.86960361067503,
|
|
"logps/rejected": -388.6978032659409,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.20194935798645,
|
|
"metrics/advantage_var": 413.9568786621094,
|
|
"regularization/lipschitz_norm": 1292.7647705078125,
|
|
"regularization/mmd": 0.19111108779907227,
|
|
"regularization/rkhs_norm": 227.51321411132812,
|
|
"regularization/w1": 1.085922360420227,
|
|
"rewards/chosen": 0.5163855919471154,
|
|
"rewards/margins": 0.8182538467555087,
|
|
"rewards/rejected": -0.3018682548083933,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 18.801280975341797,
|
|
"kl": 15.24552059173584,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34493345.18518519,
|
|
"logits/rejected": -36644008.50632911,
|
|
"logps/chosen": -472.06785300925924,
|
|
"logps/rejected": -345.8674347310127,
|
|
"loss": 0.5539,
|
|
"loss/base_kto": 3.1891977787017822,
|
|
"metrics/advantage_var": 402.05340576171875,
|
|
"regularization/lipschitz_norm": 1254.8857421875,
|
|
"regularization/mmd": 0.1878947615623474,
|
|
"regularization/rkhs_norm": 223.68423461914062,
|
|
"regularization/w1": 1.054103970527649,
|
|
"rewards/chosen": 0.5337012020158179,
|
|
"rewards/margins": 0.8231312167106113,
|
|
"rewards/rejected": -0.2894300146947933,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 18.543928146362305,
|
|
"kl": 14.762819290161133,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34066493.686746985,
|
|
"logits/rejected": -35712126.33766234,
|
|
"logps/chosen": -483.414250753012,
|
|
"logps/rejected": -387.5329748376623,
|
|
"loss": 0.5537,
|
|
"loss/base_kto": 3.1797683238983154,
|
|
"metrics/advantage_var": 413.9333190917969,
|
|
"regularization/lipschitz_norm": 1260.632080078125,
|
|
"regularization/mmd": 0.1905139535665512,
|
|
"regularization/rkhs_norm": 226.80235290527344,
|
|
"regularization/w1": 1.0589308738708496,
|
|
"rewards/chosen": 0.6176203072789204,
|
|
"rewards/margins": 0.8547411709580879,
|
|
"rewards/rejected": -0.23712086367916752,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 17.16962432861328,
|
|
"kl": 14.960174560546875,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -33796794.61682243,
|
|
"logits/rejected": -34877481.73040752,
|
|
"logps/chosen": -477.4481113707165,
|
|
"logps/rejected": -361.3479134012539,
|
|
"loss": 0.5596,
|
|
"loss/base_kto": 3.220236301422119,
|
|
"metrics/advantage_var": 375.538818359375,
|
|
"regularization/lipschitz_norm": 1273.143798828125,
|
|
"regularization/mmd": 0.18699096143245697,
|
|
"regularization/rkhs_norm": 222.60830688476562,
|
|
"regularization/w1": 1.0694407224655151,
|
|
"rewards/chosen": 0.5138200896551305,
|
|
"rewards/margins": 0.8047764014938844,
|
|
"rewards/rejected": -0.2909563118387539,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 19.583574295043945,
|
|
"kl": 13.168062210083008,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34306642.58064516,
|
|
"logits/rejected": -36551040.77575757,
|
|
"logps/chosen": -493.7453125,
|
|
"logps/rejected": -375.03804450757576,
|
|
"loss": 0.5504,
|
|
"loss/base_kto": 3.174727439880371,
|
|
"metrics/advantage_var": 382.2155456542969,
|
|
"regularization/lipschitz_norm": 1240.6024169921875,
|
|
"regularization/mmd": 0.1866205632686615,
|
|
"regularization/rkhs_norm": 222.16734313964844,
|
|
"regularization/w1": 1.042106032371521,
|
|
"rewards/chosen": 0.5039766865391885,
|
|
"rewards/margins": 0.8394093980537481,
|
|
"rewards/rejected": -0.33543271151455967,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 18.3836727142334,
|
|
"kl": 9.91015911102295,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34146055.47158218,
|
|
"logits/rejected": -35701917.91414944,
|
|
"logps/chosen": -491.77932987711216,
|
|
"logps/rejected": -386.9768978537361,
|
|
"loss": 0.5609,
|
|
"loss/base_kto": 3.171435832977295,
|
|
"metrics/advantage_var": 433.8333435058594,
|
|
"regularization/lipschitz_norm": 1331.4329833984375,
|
|
"regularization/mmd": 0.19731023907661438,
|
|
"regularization/rkhs_norm": 234.8931427001953,
|
|
"regularization/w1": 1.118403673171997,
|
|
"rewards/chosen": 0.473641109906034,
|
|
"rewards/margins": 0.8789256094216341,
|
|
"rewards/rejected": -0.40528449951560014,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 15.16122055053711,
|
|
"kl": 12.20919418334961,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33439407.429457363,
|
|
"logits/rejected": -37277218.67086614,
|
|
"logps/chosen": -493.9566860465116,
|
|
"logps/rejected": -350.99931102362206,
|
|
"loss": 0.565,
|
|
"loss/base_kto": 3.2192063331604004,
|
|
"metrics/advantage_var": 436.88458251953125,
|
|
"regularization/lipschitz_norm": 1318.0078125,
|
|
"regularization/mmd": 0.19399665296077728,
|
|
"regularization/rkhs_norm": 230.9484100341797,
|
|
"regularization/w1": 1.1071265935897827,
|
|
"rewards/chosen": 0.5183074241460756,
|
|
"rewards/margins": 0.8289919406322961,
|
|
"rewards/rejected": -0.3106845164862205,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 12.267426490783691,
|
|
"kl": 11.683924674987793,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -33398228.161550887,
|
|
"logits/rejected": -35680361.34341906,
|
|
"logps/chosen": -462.34344709208403,
|
|
"logps/rejected": -372.54387291981845,
|
|
"loss": 0.5572,
|
|
"loss/base_kto": 3.2189643383026123,
|
|
"metrics/advantage_var": 393.2417907714844,
|
|
"regularization/lipschitz_norm": 1252.6536865234375,
|
|
"regularization/mmd": 0.18667958676815033,
|
|
"regularization/rkhs_norm": 222.2375946044922,
|
|
"regularization/w1": 1.0522291660308838,
|
|
"rewards/chosen": 0.46127393288065427,
|
|
"rewards/margins": 0.8193112380113747,
|
|
"rewards/rejected": -0.3580373051307205,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 14.912087440490723,
|
|
"kl": 12.115736961364746,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34119704.6006006,
|
|
"logits/rejected": -35423252.013029315,
|
|
"logps/chosen": -475.6275807057057,
|
|
"logps/rejected": -388.89311889250814,
|
|
"loss": 0.5641,
|
|
"loss/base_kto": 3.2131574153900146,
|
|
"metrics/advantage_var": 409.0126647949219,
|
|
"regularization/lipschitz_norm": 1318.6597900390625,
|
|
"regularization/mmd": 0.19223010540008545,
|
|
"regularization/rkhs_norm": 228.84536743164062,
|
|
"regularization/w1": 1.107674241065979,
|
|
"rewards/chosen": 0.4986439839497701,
|
|
"rewards/margins": 0.8328326378522029,
|
|
"rewards/rejected": -0.3341886539024328,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 14.157485961914062,
|
|
"kl": 11.526637077331543,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35234653.433715224,
|
|
"logits/rejected": -35704262.60089686,
|
|
"logps/chosen": -489.0647504091653,
|
|
"logps/rejected": -368.95975803437966,
|
|
"loss": 0.5554,
|
|
"loss/base_kto": 3.2270355224609375,
|
|
"metrics/advantage_var": 381.1188659667969,
|
|
"regularization/lipschitz_norm": 1226.8309326171875,
|
|
"regularization/mmd": 0.18547473847866058,
|
|
"regularization/rkhs_norm": 220.8032684326172,
|
|
"regularization/w1": 1.0305380821228027,
|
|
"rewards/chosen": 0.481105223810224,
|
|
"rewards/margins": 0.7920113760080005,
|
|
"rewards/rejected": -0.3109061521977765,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 15.411792755126953,
|
|
"kl": 12.602194786071777,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34904763.241274655,
|
|
"logits/rejected": -35494422.26086956,
|
|
"logps/chosen": -484.5643019726859,
|
|
"logps/rejected": -387.20959138486313,
|
|
"loss": 0.5531,
|
|
"loss/base_kto": 3.169900894165039,
|
|
"metrics/advantage_var": 401.9240417480469,
|
|
"regularization/lipschitz_norm": 1265.5244140625,
|
|
"regularization/mmd": 0.19208136200904846,
|
|
"regularization/rkhs_norm": 228.66831970214844,
|
|
"regularization/w1": 1.0630406141281128,
|
|
"rewards/chosen": 0.4658034560532056,
|
|
"rewards/margins": 0.8653867412267537,
|
|
"rewards/rejected": -0.3995832851735482,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 17.928422927856445,
|
|
"kl": 12.969223022460938,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -33836700.37829457,
|
|
"logits/rejected": -35256174.86614173,
|
|
"logps/chosen": -483.8582364341085,
|
|
"logps/rejected": -375.6524114173228,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.20304799079895,
|
|
"metrics/advantage_var": 395.5731506347656,
|
|
"regularization/lipschitz_norm": 1245.0765380859375,
|
|
"regularization/mmd": 0.18924403190612793,
|
|
"regularization/rkhs_norm": 225.29051208496094,
|
|
"regularization/w1": 1.0458643436431885,
|
|
"rewards/chosen": 0.47001281265140504,
|
|
"rewards/margins": 0.7970668619255192,
|
|
"rewards/rejected": -0.3270540492741142,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 14.488730430603027,
|
|
"kl": 14.978285789489746,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35953340.88540032,
|
|
"logits/rejected": -37100988.31726283,
|
|
"logps/chosen": -486.3620977237049,
|
|
"logps/rejected": -371.02148133748057,
|
|
"loss": 0.5518,
|
|
"loss/base_kto": 3.1891443729400635,
|
|
"metrics/advantage_var": 380.630859375,
|
|
"regularization/lipschitz_norm": 1236.129150390625,
|
|
"regularization/mmd": 0.18714351952075958,
|
|
"regularization/rkhs_norm": 222.78990173339844,
|
|
"regularization/w1": 1.0383485555648804,
|
|
"rewards/chosen": 0.500652127587814,
|
|
"rewards/margins": 0.8110233959951818,
|
|
"rewards/rejected": -0.3103712684073678,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 17.652151107788086,
|
|
"kl": 12.483137130737305,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33551701.333333332,
|
|
"logits/rejected": -35994428.18808778,
|
|
"logps/chosen": -460.87032710280374,
|
|
"logps/rejected": -376.2286931818182,
|
|
"loss": 0.5484,
|
|
"loss/base_kto": 3.1624042987823486,
|
|
"metrics/advantage_var": 377.28350830078125,
|
|
"regularization/lipschitz_norm": 1237.3677978515625,
|
|
"regularization/mmd": 0.18508975207805634,
|
|
"regularization/rkhs_norm": 220.3449249267578,
|
|
"regularization/w1": 1.03938889503479,
|
|
"rewards/chosen": 0.5844000225126558,
|
|
"rewards/margins": 0.8646310798022816,
|
|
"rewards/rejected": -0.2802310572896258,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 14.638246536254883,
|
|
"kl": 14.82068157196045,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34098961.788519636,
|
|
"logits/rejected": -35118249.00970874,
|
|
"logps/chosen": -477.7122356495468,
|
|
"logps/rejected": -351.9229874595469,
|
|
"loss": 0.5627,
|
|
"loss/base_kto": 3.2100231647491455,
|
|
"metrics/advantage_var": 428.5296325683594,
|
|
"regularization/lipschitz_norm": 1304.7535400390625,
|
|
"regularization/mmd": 0.19541814923286438,
|
|
"regularization/rkhs_norm": 232.64065551757812,
|
|
"regularization/w1": 1.095992922782898,
|
|
"rewards/chosen": 0.5419090247946563,
|
|
"rewards/margins": 0.8043162967878552,
|
|
"rewards/rejected": -0.26240727199319885,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.750174522399902,
|
|
"kl": 13.8255033493042,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34473980.7232,
|
|
"logits/rejected": -35339681.41679389,
|
|
"logps/chosen": -478.5715,
|
|
"logps/rejected": -385.5200143129771,
|
|
"loss": 0.5544,
|
|
"loss/base_kto": 3.1820104122161865,
|
|
"metrics/advantage_var": 420.1051330566406,
|
|
"regularization/lipschitz_norm": 1263.3740234375,
|
|
"regularization/mmd": 0.1922682672739029,
|
|
"regularization/rkhs_norm": 228.8907928466797,
|
|
"regularization/w1": 1.0612342357635498,
|
|
"rewards/chosen": 0.544025732421875,
|
|
"rewards/margins": 0.8477923936217796,
|
|
"rewards/rejected": -0.3037666611999046,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 15.82647705078125,
|
|
"kl": 12.365191459655762,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -36220718.2848,
|
|
"logits/rejected": -36358015.80458015,
|
|
"logps/chosen": -504.9765,
|
|
"logps/rejected": -385.2235687022901,
|
|
"loss": 0.5645,
|
|
"loss/base_kto": 3.17185378074646,
|
|
"metrics/advantage_var": 434.1415100097656,
|
|
"regularization/lipschitz_norm": 1367.070068359375,
|
|
"regularization/mmd": 0.19565115869045258,
|
|
"regularization/rkhs_norm": 232.9180450439453,
|
|
"regularization/w1": 1.1483389139175415,
|
|
"rewards/chosen": 0.55656669921875,
|
|
"rewards/margins": 0.869565347126968,
|
|
"rewards/rejected": -0.312998647908218,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 18.393125534057617,
|
|
"kl": 13.640936851501465,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34971681.08562197,
|
|
"logits/rejected": -35987747.24357035,
|
|
"logps/chosen": -480.092084006462,
|
|
"logps/rejected": -379.8142492435704,
|
|
"loss": 0.5515,
|
|
"loss/base_kto": 3.168067693710327,
|
|
"metrics/advantage_var": 382.4317321777344,
|
|
"regularization/lipschitz_norm": 1257.4322509765625,
|
|
"regularization/mmd": 0.1876767873764038,
|
|
"regularization/rkhs_norm": 223.4247589111328,
|
|
"regularization/w1": 1.0562430620193481,
|
|
"rewards/chosen": 0.5432558321606169,
|
|
"rewards/margins": 0.8510814718016674,
|
|
"rewards/rejected": -0.3078256396410505,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 12.669471740722656,
|
|
"kl": 13.345707893371582,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35006115.87136294,
|
|
"logits/rejected": -35269734.88995215,
|
|
"logps/chosen": -489.25981049004594,
|
|
"logps/rejected": -395.77606658692184,
|
|
"loss": 0.561,
|
|
"loss/base_kto": 3.112929344177246,
|
|
"metrics/advantage_var": 460.5668029785156,
|
|
"regularization/lipschitz_norm": 1393.0103759765625,
|
|
"regularization/mmd": 0.20509754121303558,
|
|
"regularization/rkhs_norm": 244.16372680664062,
|
|
"regularization/w1": 1.1701288223266602,
|
|
"rewards/chosen": 0.5679244644609016,
|
|
"rewards/margins": 0.9302929614424358,
|
|
"rewards/rejected": -0.3623684969815341,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 37.84089660644531,
|
|
"kl": 13.3887357711792,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34525732.74641148,
|
|
"logits/rejected": -34301378.05819295,
|
|
"logps/chosen": -468.1487240829346,
|
|
"logps/rejected": -367.65376148545175,
|
|
"loss": 0.551,
|
|
"loss/base_kto": 3.2018883228302,
|
|
"metrics/advantage_var": 380.1961364746094,
|
|
"regularization/lipschitz_norm": 1217.9705810546875,
|
|
"regularization/mmd": 0.18332134187221527,
|
|
"regularization/rkhs_norm": 218.2397003173828,
|
|
"regularization/w1": 1.0230953693389893,
|
|
"rewards/chosen": 0.5104755115661135,
|
|
"rewards/margins": 0.8121196659497276,
|
|
"rewards/rejected": -0.3016441543836141,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 21.137928009033203,
|
|
"kl": 13.122614860534668,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35794062.73939394,
|
|
"logits/rejected": -35638503.22580645,
|
|
"logps/chosen": -483.5817234848485,
|
|
"logps/rejected": -371.6561491935484,
|
|
"loss": 0.5559,
|
|
"loss/base_kto": 3.177448034286499,
|
|
"metrics/advantage_var": 399.42236328125,
|
|
"regularization/lipschitz_norm": 1283.5113525390625,
|
|
"regularization/mmd": 0.1913510113954544,
|
|
"regularization/rkhs_norm": 227.798828125,
|
|
"regularization/w1": 1.0781495571136475,
|
|
"rewards/chosen": 0.531743876139323,
|
|
"rewards/margins": 0.8441938420777679,
|
|
"rewards/rejected": -0.31244996593844504,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 13.346651077270508,
|
|
"kl": 14.126541137695312,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34391896.55345912,
|
|
"logits/rejected": -34710617.04347826,
|
|
"logps/chosen": -478.78803066037733,
|
|
"logps/rejected": -377.1985636645963,
|
|
"loss": 0.5567,
|
|
"loss/base_kto": 3.2097671031951904,
|
|
"metrics/advantage_var": 383.3074645996094,
|
|
"regularization/lipschitz_norm": 1259.958740234375,
|
|
"regularization/mmd": 0.18560345470905304,
|
|
"regularization/rkhs_norm": 220.9564666748047,
|
|
"regularization/w1": 1.0583652257919312,
|
|
"rewards/chosen": 0.509803628021816,
|
|
"rewards/margins": 0.8211476562814725,
|
|
"rewards/rejected": -0.31134402825965646,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 16.06229591369629,
|
|
"kl": 13.415672302246094,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34598424.075235106,
|
|
"logits/rejected": -35016576.3987539,
|
|
"logps/chosen": -485.39371081504703,
|
|
"logps/rejected": -354.51255841121497,
|
|
"loss": 0.5592,
|
|
"loss/base_kto": 3.1724281311035156,
|
|
"metrics/advantage_var": 424.5101623535156,
|
|
"regularization/lipschitz_norm": 1316.0338134765625,
|
|
"regularization/mmd": 0.19575339555740356,
|
|
"regularization/rkhs_norm": 233.03977966308594,
|
|
"regularization/w1": 1.1054683923721313,
|
|
"rewards/chosen": 0.5497104082735355,
|
|
"rewards/margins": 0.8501877184066644,
|
|
"rewards/rejected": -0.3004773101331289,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 15.011834144592285,
|
|
"kl": 13.388748168945312,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34554275.275590554,
|
|
"logits/rejected": -35839434.81550387,
|
|
"logps/chosen": -486.4338582677165,
|
|
"logps/rejected": -381.23968023255816,
|
|
"loss": 0.5837,
|
|
"loss/base_kto": 3.2260849475860596,
|
|
"metrics/advantage_var": 755.8993530273438,
|
|
"regularization/lipschitz_norm": 1464.78369140625,
|
|
"regularization/mmd": 0.21273119747638702,
|
|
"regularization/rkhs_norm": 253.2514190673828,
|
|
"regularization/w1": 1.2304182052612305,
|
|
"rewards/chosen": 0.46978519469734253,
|
|
"rewards/margins": 0.7591148299626042,
|
|
"rewards/rejected": -0.2893296352652616,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_loss": 0.587135434356584,
|
|
"train_runtime": 11136.5791,
|
|
"train_samples_per_second": 13.309,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|