2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 18.355131149291992,
|
||
|
|
"kl": 8.59229564666748,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -35677106.2278481,
|
||
|
|
"logits/rejected": -37380674.37037037,
|
||
|
|
"logps/chosen": -501.69076344936707,
|
||
|
|
"logps/rejected": -375.19931520061726,
|
||
|
|
"loss": 0.5987,
|
||
|
|
"loss/base_kto": 3.899252414703369,
|
||
|
|
"metrics/advantage_var": 239.6805419921875,
|
||
|
|
"regularization/mmd": 0.8906540870666504,
|
||
|
|
"regularization/rkhs_norm": 171.60964965820312,
|
||
|
|
"rewards/chosen": 0.050869072539896906,
|
||
|
|
"rewards/margins": 0.09678698972084128,
|
||
|
|
"rewards/rejected": -0.04591791718094437,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 14.383731842041016,
|
||
|
|
"kl": 18.936019897460938,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -35806462.02773498,
|
||
|
|
"logits/rejected": -37790854.69413629,
|
||
|
|
"logps/chosen": -491.8996533127889,
|
||
|
|
"logps/rejected": -360.08897087955626,
|
||
|
|
"loss": 0.589,
|
||
|
|
"loss/base_kto": 3.903554916381836,
|
||
|
|
"metrics/advantage_var": 188.86729431152344,
|
||
|
|
"regularization/mmd": 0.8083402514457703,
|
||
|
|
"regularization/rkhs_norm": 155.7495574951172,
|
||
|
|
"rewards/chosen": 0.26754955885406634,
|
||
|
|
"rewards/margins": 0.07950186263619541,
|
||
|
|
"rewards/rejected": 0.18804769621787093,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 20.89337921142578,
|
||
|
|
"kl": 3.5445687770843506,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -34724693.333333336,
|
||
|
|
"logits/rejected": -35590390.63414634,
|
||
|
|
"logps/chosen": -470.2059294871795,
|
||
|
|
"logps/rejected": -364.9865186737805,
|
||
|
|
"loss": 0.576,
|
||
|
|
"loss/base_kto": 3.835980176925659,
|
||
|
|
"metrics/advantage_var": 184.19546508789062,
|
||
|
|
"regularization/mmd": 0.7723119258880615,
|
||
|
|
"regularization/rkhs_norm": 148.80770874023438,
|
||
|
|
"rewards/chosen": 0.042210071514814325,
|
||
|
|
"rewards/margins": 0.16762212860054937,
|
||
|
|
"rewards/rejected": -0.12541205708573505,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 16.099123001098633,
|
||
|
|
"kl": 7.2627739906311035,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -37767945.27244095,
|
||
|
|
"logits/rejected": -37982627.1255814,
|
||
|
|
"logps/chosen": -505.8322342519685,
|
||
|
|
"logps/rejected": -385.1265988372093,
|
||
|
|
"loss": 0.5774,
|
||
|
|
"loss/base_kto": 3.831442356109619,
|
||
|
|
"metrics/advantage_var": 175.14170837402344,
|
||
|
|
"regularization/mmd": 0.7881031036376953,
|
||
|
|
"regularization/rkhs_norm": 151.85031127929688,
|
||
|
|
"rewards/chosen": 0.11390412097840798,
|
||
|
|
"rewards/margins": 0.16512695915102488,
|
||
|
|
"rewards/rejected": -0.051222838172616886,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 15.046182632446289,
|
||
|
|
"kl": 9.304082870483398,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -36750334.43185299,
|
||
|
|
"logits/rejected": -38120995.11323764,
|
||
|
|
"logps/chosen": -490.5138782542113,
|
||
|
|
"logps/rejected": -373.44328149920256,
|
||
|
|
"loss": 0.5912,
|
||
|
|
"loss/base_kto": 3.834928274154663,
|
||
|
|
"metrics/advantage_var": 241.1676025390625,
|
||
|
|
"regularization/mmd": 0.8943428993225098,
|
||
|
|
"regularization/rkhs_norm": 172.32041931152344,
|
||
|
|
"rewards/chosen": 0.07395970109410892,
|
||
|
|
"rewards/margins": 0.18977724413606217,
|
||
|
|
"rewards/rejected": -0.11581754304195326,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 14.61628246307373,
|
||
|
|
"kl": 12.464029312133789,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -37586362.43053435,
|
||
|
|
"logits/rejected": -37443043.328,
|
||
|
|
"logps/chosen": -491.85190839694656,
|
||
|
|
"logps/rejected": -361.0565,
|
||
|
|
"loss": 0.5865,
|
||
|
|
"loss/base_kto": 3.87249755859375,
|
||
|
|
"metrics/advantage_var": 199.67323303222656,
|
||
|
|
"regularization/mmd": 0.8192858099937439,
|
||
|
|
"regularization/rkhs_norm": 157.85855102539062,
|
||
|
|
"rewards/chosen": 0.2136779086280415,
|
||
|
|
"rewards/margins": 0.11438920013194775,
|
||
|
|
"rewards/rejected": 0.09928870849609375,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 14.474100112915039,
|
||
|
|
"kl": 12.758808135986328,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -35743580.66257669,
|
||
|
|
"logits/rejected": -38627249.73248408,
|
||
|
|
"logps/chosen": -485.7898773006135,
|
||
|
|
"logps/rejected": -384.0504329219745,
|
||
|
|
"loss": 0.5778,
|
||
|
|
"loss/base_kto": 3.8082337379455566,
|
||
|
|
"metrics/advantage_var": 186.1129150390625,
|
||
|
|
"regularization/mmd": 0.8143102526664734,
|
||
|
|
"regularization/rkhs_norm": 156.8998565673828,
|
||
|
|
"rewards/chosen": 0.16835581305568204,
|
||
|
|
"rewards/margins": 0.155821749722523,
|
||
|
|
"rewards/rejected": 0.01253406333315904,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 18.011737823486328,
|
||
|
|
"kl": 13.618170738220215,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -37305216.575712144,
|
||
|
|
"logits/rejected": -36834078.48613377,
|
||
|
|
"logps/chosen": -495.27886056971516,
|
||
|
|
"logps/rejected": -373.70554649265904,
|
||
|
|
"loss": 0.5813,
|
||
|
|
"loss/base_kto": 3.7884018421173096,
|
||
|
|
"metrics/advantage_var": 216.3523712158203,
|
||
|
|
"regularization/mmd": 0.8617948889732361,
|
||
|
|
"regularization/rkhs_norm": 166.049072265625,
|
||
|
|
"rewards/chosen": 0.30625135430331707,
|
||
|
|
"rewards/margins": 0.18292163163591546,
|
||
|
|
"rewards/rejected": 0.12332972266740161,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 17.904626846313477,
|
||
|
|
"kl": 17.634511947631836,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -35702719.90461997,
|
||
|
|
"logits/rejected": -36885959.67159277,
|
||
|
|
"logps/chosen": -466.5516952309985,
|
||
|
|
"logps/rejected": -359.201867816092,
|
||
|
|
"loss": 0.5749,
|
||
|
|
"loss/base_kto": 3.762676239013672,
|
||
|
|
"metrics/advantage_var": 196.781494140625,
|
||
|
|
"regularization/mmd": 0.8366878628730774,
|
||
|
|
"regularization/rkhs_norm": 161.21153259277344,
|
||
|
|
"rewards/chosen": 0.31675417767907044,
|
||
|
|
"rewards/margins": 0.21232553088360656,
|
||
|
|
"rewards/rejected": 0.10442864679546388,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 13.703529357910156,
|
||
|
|
"kl": 15.65002727508545,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -37641179.537091985,
|
||
|
|
"logits/rejected": -39380167.39273927,
|
||
|
|
"logps/chosen": -488.34597551928783,
|
||
|
|
"logps/rejected": -382.0031456270627,
|
||
|
|
"loss": 0.5825,
|
||
|
|
"loss/base_kto": 3.7953789234161377,
|
||
|
|
"metrics/advantage_var": 218.0191650390625,
|
||
|
|
"regularization/mmd": 0.8642555475234985,
|
||
|
|
"regularization/rkhs_norm": 166.52322387695312,
|
||
|
|
"rewards/chosen": 0.28980716965672526,
|
||
|
|
"rewards/margins": 0.1635241472199601,
|
||
|
|
"rewards/rejected": 0.12628302243676517,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 24.622936248779297,
|
||
|
|
"kl": 11.670283317565918,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37326730.5767285,
|
||
|
|
"logits/rejected": -38053563.06259097,
|
||
|
|
"logps/chosen": -479.42780354131537,
|
||
|
|
"logps/rejected": -380.7495906113537,
|
||
|
|
"loss": 0.5852,
|
||
|
|
"loss/base_kto": 3.7580573558807373,
|
||
|
|
"metrics/advantage_var": 249.5292205810547,
|
||
|
|
"regularization/mmd": 0.9234267473220825,
|
||
|
|
"regularization/rkhs_norm": 177.92422485351562,
|
||
|
|
"rewards/chosen": 0.13914695598222754,
|
||
|
|
"rewards/margins": 0.2099495529614937,
|
||
|
|
"rewards/rejected": -0.07080259697926618,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 15.743287086486816,
|
||
|
|
"kl": 8.136269569396973,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -35933149.9207607,
|
||
|
|
"logits/rejected": -36145742.10169491,
|
||
|
|
"logps/chosen": -466.4460677496038,
|
||
|
|
"logps/rejected": -377.88212634822804,
|
||
|
|
"loss": 0.5811,
|
||
|
|
"loss/base_kto": 3.7892513275146484,
|
||
|
|
"metrics/advantage_var": 212.422607421875,
|
||
|
|
"regularization/mmd": 0.8593311309814453,
|
||
|
|
"regularization/rkhs_norm": 165.5743865966797,
|
||
|
|
"rewards/chosen": 0.029862895064799798,
|
||
|
|
"rewards/margins": 0.2177220761441647,
|
||
|
|
"rewards/rejected": -0.1878591810793649,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 16.04319953918457,
|
||
|
|
"kl": 6.755073547363281,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -35730253.35825545,
|
||
|
|
"logits/rejected": -35731314.75862069,
|
||
|
|
"logps/chosen": -479.7824182242991,
|
||
|
|
"logps/rejected": -379.6119710031348,
|
||
|
|
"loss": 0.5825,
|
||
|
|
"loss/base_kto": 3.7791366577148438,
|
||
|
|
"metrics/advantage_var": 230.1333465576172,
|
||
|
|
"regularization/mmd": 0.880833625793457,
|
||
|
|
"regularization/rkhs_norm": 169.7174835205078,
|
||
|
|
"rewards/chosen": 0.06319318753536617,
|
||
|
|
"rewards/margins": 0.20973908798186036,
|
||
|
|
"rewards/rejected": -0.14654590044649418,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 14.483595848083496,
|
||
|
|
"kl": 8.073613166809082,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -35130597.945072696,
|
||
|
|
"logits/rejected": -37963147.03782148,
|
||
|
|
"logps/chosen": -505.0274131663974,
|
||
|
|
"logps/rejected": -364.7469270045386,
|
||
|
|
"loss": 0.5763,
|
||
|
|
"loss/base_kto": 3.7302238941192627,
|
||
|
|
"metrics/advantage_var": 218.50735473632812,
|
||
|
|
"regularization/mmd": 0.880548894405365,
|
||
|
|
"regularization/rkhs_norm": 169.66261291503906,
|
||
|
|
"rewards/chosen": 0.14740888752729325,
|
||
|
|
"rewards/margins": 0.2510804938248656,
|
||
|
|
"rewards/rejected": -0.10367160629757233,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 12.326614379882812,
|
||
|
|
"kl": 6.938302516937256,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -35278198.035928145,
|
||
|
|
"logits/rejected": -37310162.823529415,
|
||
|
|
"logps/chosen": -463.1574663173653,
|
||
|
|
"logps/rejected": -374.53079044117646,
|
||
|
|
"loss": 0.5839,
|
||
|
|
"loss/base_kto": 3.781132459640503,
|
||
|
|
"metrics/advantage_var": 229.2920379638672,
|
||
|
|
"regularization/mmd": 0.8904251456260681,
|
||
|
|
"regularization/rkhs_norm": 171.56553649902344,
|
||
|
|
"rewards/chosen": 0.06592190337038326,
|
||
|
|
"rewards/margins": 0.21471668841493494,
|
||
|
|
"rewards/rejected": -0.14879478504455168,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 14.155945777893066,
|
||
|
|
"kl": 13.264572143554688,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -36832059.14054927,
|
||
|
|
"logits/rejected": -37249687.04387292,
|
||
|
|
"logps/chosen": -522.9134693053312,
|
||
|
|
"logps/rejected": -394.5444875189107,
|
||
|
|
"loss": 0.5837,
|
||
|
|
"loss/base_kto": 3.7714180946350098,
|
||
|
|
"metrics/advantage_var": 222.8312225341797,
|
||
|
|
"regularization/mmd": 0.8983078002929688,
|
||
|
|
"regularization/rkhs_norm": 173.0843505859375,
|
||
|
|
"rewards/chosen": 0.2278742874958981,
|
||
|
|
"rewards/margins": 0.1922227962989959,
|
||
|
|
"rewards/rejected": 0.03565149119690218,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 14.930360794067383,
|
||
|
|
"kl": 5.002163887023926,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -34289873.38019169,
|
||
|
|
"logits/rejected": -35831310.09174312,
|
||
|
|
"logps/chosen": -471.3543330670926,
|
||
|
|
"logps/rejected": -364.5137136850153,
|
||
|
|
"loss": 0.5842,
|
||
|
|
"loss/base_kto": 3.77946400642395,
|
||
|
|
"metrics/advantage_var": 251.5850067138672,
|
||
|
|
"regularization/mmd": 0.8943830728530884,
|
||
|
|
"regularization/rkhs_norm": 172.32815551757812,
|
||
|
|
"rewards/chosen": 0.019760937736438104,
|
||
|
|
"rewards/margins": 0.21380272358031355,
|
||
|
|
"rewards/rejected": -0.19404178584387544,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 17.12563133239746,
|
||
|
|
"kl": 5.303887367248535,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -34518550.33279483,
|
||
|
|
"logits/rejected": -35829217.79122542,
|
||
|
|
"logps/chosen": -496.31078352180936,
|
||
|
|
"logps/rejected": -365.0048458774584,
|
||
|
|
"loss": 0.5784,
|
||
|
|
"loss/base_kto": 3.7350640296936035,
|
||
|
|
"metrics/advantage_var": 225.28868103027344,
|
||
|
|
"regularization/mmd": 0.8921697735786438,
|
||
|
|
"regularization/rkhs_norm": 171.90170288085938,
|
||
|
|
"rewards/chosen": 0.03668622400533402,
|
||
|
|
"rewards/margins": 0.2573953971183966,
|
||
|
|
"rewards/rejected": -0.2207091731130626,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 16.593212127685547,
|
||
|
|
"kl": 5.176967620849609,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -35902288.77864992,
|
||
|
|
"logits/rejected": -36061680.87091757,
|
||
|
|
"logps/chosen": -482.09399529042383,
|
||
|
|
"logps/rejected": -355.24023133748057,
|
||
|
|
"loss": 0.5873,
|
||
|
|
"loss/base_kto": 3.784255266189575,
|
||
|
|
"metrics/advantage_var": 238.3084716796875,
|
||
|
|
"regularization/mmd": 0.9138066172599792,
|
||
|
|
"regularization/rkhs_norm": 176.0706329345703,
|
||
|
|
"rewards/chosen": 0.01627715303909086,
|
||
|
|
"rewards/margins": 0.2028148763727077,
|
||
|
|
"rewards/rejected": -0.18653772333361684,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 15.823249816894531,
|
||
|
|
"kl": 4.430182933807373,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -36196202.463492066,
|
||
|
|
"logits/rejected": -36116335.064615384,
|
||
|
|
"logps/chosen": -482.8965277777778,
|
||
|
|
"logps/rejected": -389.58752403846154,
|
||
|
|
"loss": 0.5814,
|
||
|
|
"loss/base_kto": 3.7527215480804443,
|
||
|
|
"metrics/advantage_var": 236.9881134033203,
|
||
|
|
"regularization/mmd": 0.8983505368232727,
|
||
|
|
"regularization/rkhs_norm": 173.0925750732422,
|
||
|
|
"rewards/chosen": -0.05178233555385044,
|
||
|
|
"rewards/margins": 0.2334475322346111,
|
||
|
|
"rewards/rejected": -0.28522986778846154,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 13.52650260925293,
|
||
|
|
"kl": 8.73814582824707,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -37322299.760252364,
|
||
|
|
"logits/rejected": -38061487.15789474,
|
||
|
|
"logps/chosen": -478.4477523659306,
|
||
|
|
"logps/rejected": -358.9422407120743,
|
||
|
|
"loss": 0.5795,
|
||
|
|
"loss/base_kto": 3.7514564990997314,
|
||
|
|
"metrics/advantage_var": 219.9439697265625,
|
||
|
|
"regularization/mmd": 0.8847070932388306,
|
||
|
|
"regularization/rkhs_norm": 170.4637908935547,
|
||
|
|
"rewards/chosen": 0.07475940210962145,
|
||
|
|
"rewards/margins": 0.2192280343854656,
|
||
|
|
"rewards/rejected": -0.14446863227584414,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 14.851932525634766,
|
||
|
|
"kl": 10.129216194152832,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -35613759.209876545,
|
||
|
|
"logits/rejected": -36460612.05063291,
|
||
|
|
"logps/chosen": -454.60947145061726,
|
||
|
|
"logps/rejected": -376.8286194620253,
|
||
|
|
"loss": 0.5739,
|
||
|
|
"loss/base_kto": 3.726673126220703,
|
||
|
|
"metrics/advantage_var": 211.911865234375,
|
||
|
|
"regularization/mmd": 0.8641713261604309,
|
||
|
|
"regularization/rkhs_norm": 166.5070037841797,
|
||
|
|
"rewards/chosen": 0.17373162728768807,
|
||
|
|
"rewards/margins": 0.23917970017988915,
|
||
|
|
"rewards/rejected": -0.0654480728922011,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 16.98316764831543,
|
||
|
|
"kl": 11.698954582214355,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -35815370.845801525,
|
||
|
|
"logits/rejected": -36148592.64,
|
||
|
|
"logps/chosen": -465.42123091603054,
|
||
|
|
"logps/rejected": -356.472725,
|
||
|
|
"loss": 0.578,
|
||
|
|
"loss/base_kto": 3.681044816970825,
|
||
|
|
"metrics/advantage_var": 263.0597839355469,
|
||
|
|
"regularization/mmd": 0.9432243704795837,
|
||
|
|
"regularization/rkhs_norm": 181.73878479003906,
|
||
|
|
"rewards/chosen": 0.2568790348431536,
|
||
|
|
"rewards/margins": 0.3083677311322161,
|
||
|
|
"rewards/rejected": -0.0514886962890625,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 18.69508934020996,
|
||
|
|
"kl": 14.229891777038574,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -37249244.21505377,
|
||
|
|
"logits/rejected": -37633168.89030207,
|
||
|
|
"logps/chosen": -494.7801459293395,
|
||
|
|
"logps/rejected": -388.1153616852146,
|
||
|
|
"loss": 0.5859,
|
||
|
|
"loss/base_kto": 3.6975953578948975,
|
||
|
|
"metrics/advantage_var": 285.19891357421875,
|
||
|
|
"regularization/mmd": 0.9894009828567505,
|
||
|
|
"regularization/rkhs_norm": 190.6360321044922,
|
||
|
|
"rewards/chosen": 0.29050962386592744,
|
||
|
|
"rewards/margins": 0.29268515360062536,
|
||
|
|
"rewards/rejected": -0.0021755297346979334,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 13.925016403198242,
|
||
|
|
"kl": 16.193500518798828,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -35656988.706605226,
|
||
|
|
"logits/rejected": -37467951.61844197,
|
||
|
|
"logps/chosen": -465.0112807219662,
|
||
|
|
"logps/rejected": -384.5026331478537,
|
||
|
|
"loss": 0.5786,
|
||
|
|
"loss/base_kto": 3.7520835399627686,
|
||
|
|
"metrics/advantage_var": 226.3187255859375,
|
||
|
|
"regularization/mmd": 0.876621425151825,
|
||
|
|
"regularization/rkhs_norm": 168.90586853027344,
|
||
|
|
"rewards/chosen": 0.2240577381327405,
|
||
|
|
"rewards/margins": 0.21612428372532297,
|
||
|
|
"rewards/rejected": 0.007933454407417528,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 15.746620178222656,
|
||
|
|
"kl": 9.902660369873047,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -36418136.83280757,
|
||
|
|
"logits/rejected": -37338102.489164084,
|
||
|
|
"logps/chosen": -465.4365634858044,
|
||
|
|
"logps/rejected": -379.03957043343655,
|
||
|
|
"loss": 0.5871,
|
||
|
|
"loss/base_kto": 3.785489797592163,
|
||
|
|
"metrics/advantage_var": 242.4727783203125,
|
||
|
|
"regularization/mmd": 0.9116144180297852,
|
||
|
|
"regularization/rkhs_norm": 175.6482391357422,
|
||
|
|
"rewards/chosen": 0.1177103060653157,
|
||
|
|
"rewards/margins": 0.18170398443876468,
|
||
|
|
"rewards/rejected": -0.06399367837344899,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 15.057689666748047,
|
||
|
|
"kl": 15.38354206085205,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -36851674.59360731,
|
||
|
|
"logits/rejected": -36642947.492776886,
|
||
|
|
"logps/chosen": -500.0726788432268,
|
||
|
|
"logps/rejected": -359.93649678972713,
|
||
|
|
"loss": 0.5677,
|
||
|
|
"loss/base_kto": 3.6362855434417725,
|
||
|
|
"metrics/advantage_var": 236.0905303955078,
|
||
|
|
"regularization/mmd": 0.9051849246025085,
|
||
|
|
"regularization/rkhs_norm": 174.40945434570312,
|
||
|
|
"rewards/chosen": 0.3106789784888699,
|
||
|
|
"rewards/margins": 0.3464053420367936,
|
||
|
|
"rewards/rejected": -0.03572636354792367,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 14.786884307861328,
|
||
|
|
"kl": 16.252817153930664,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -36130636.29652043,
|
||
|
|
"logits/rejected": -36427646.96607431,
|
||
|
|
"logps/chosen": -496.38705559757943,
|
||
|
|
"logps/rejected": -385.2689317447496,
|
||
|
|
"loss": 0.5856,
|
||
|
|
"loss/base_kto": 3.7102839946746826,
|
||
|
|
"metrics/advantage_var": 268.5868225097656,
|
||
|
|
"regularization/mmd": 0.9748560190200806,
|
||
|
|
"regularization/rkhs_norm": 187.8335418701172,
|
||
|
|
"rewards/chosen": 0.3056913020931827,
|
||
|
|
"rewards/margins": 0.2599889825551094,
|
||
|
|
"rewards/rejected": 0.04570231953807331,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 14.099896430969238,
|
||
|
|
"kl": 19.939496994018555,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36829003.85185185,
|
||
|
|
"logits/rejected": -37800435.037974685,
|
||
|
|
"logps/chosen": -504.62191358024694,
|
||
|
|
"logps/rejected": -367.6693285205696,
|
||
|
|
"loss": 0.5723,
|
||
|
|
"loss/base_kto": 3.695143938064575,
|
||
|
|
"metrics/advantage_var": 217.6491241455078,
|
||
|
|
"regularization/mmd": 0.8835013508796692,
|
||
|
|
"regularization/rkhs_norm": 170.23147583007812,
|
||
|
|
"rewards/chosen": 0.3421223251907914,
|
||
|
|
"rewards/margins": 0.23950373349589918,
|
||
|
|
"rewards/rejected": 0.10261859169489221,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 16.136192321777344,
|
||
|
|
"kl": 9.32910442352295,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -36956643.55555555,
|
||
|
|
"logits/rejected": -37741010.63291139,
|
||
|
|
"logps/chosen": -524.8196855709876,
|
||
|
|
"logps/rejected": -366.2413963607595,
|
||
|
|
"loss": 0.5775,
|
||
|
|
"loss/base_kto": 3.747807264328003,
|
||
|
|
"metrics/advantage_var": 220.1378173828125,
|
||
|
|
"regularization/mmd": 0.8723760843276978,
|
||
|
|
"regularization/rkhs_norm": 168.087890625,
|
||
|
|
"rewards/chosen": 0.1329013447702667,
|
||
|
|
"rewards/margins": 0.23484869460833036,
|
||
|
|
"rewards/rejected": -0.10194734983806368,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 21.093189239501953,
|
||
|
|
"kl": 10.337899208068848,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -34430201.952755906,
|
||
|
|
"logits/rejected": -36404139.85736434,
|
||
|
|
"logps/chosen": -476.6104330708661,
|
||
|
|
"logps/rejected": -383.88822674418606,
|
||
|
|
"loss": 0.5831,
|
||
|
|
"loss/base_kto": 3.7550950050354004,
|
||
|
|
"metrics/advantage_var": 256.0594787597656,
|
||
|
|
"regularization/mmd": 0.9093719720840454,
|
||
|
|
"regularization/rkhs_norm": 175.21617126464844,
|
||
|
|
"rewards/chosen": 0.10829953171136811,
|
||
|
|
"rewards/margins": 0.18434801160477896,
|
||
|
|
"rewards/rejected": -0.07604847989341085,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 16.082395553588867,
|
||
|
|
"kl": 8.053885459899902,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -35425285.04433498,
|
||
|
|
"logits/rejected": -37677919.76154993,
|
||
|
|
"logps/chosen": -481.14706486042695,
|
||
|
|
"logps/rejected": -369.09719634873323,
|
||
|
|
"loss": 0.5824,
|
||
|
|
"loss/base_kto": 3.744581937789917,
|
||
|
|
"metrics/advantage_var": 239.15542602539062,
|
||
|
|
"regularization/mmd": 0.9146170020103455,
|
||
|
|
"regularization/rkhs_norm": 176.22679138183594,
|
||
|
|
"rewards/chosen": 0.05715729411208179,
|
||
|
|
"rewards/margins": 0.23221122745041195,
|
||
|
|
"rewards/rejected": -0.17505393333833016,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 14.41285228729248,
|
||
|
|
"kl": 9.332486152648926,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -34698492.01246106,
|
||
|
|
"logits/rejected": -35375726.7460815,
|
||
|
|
"logps/chosen": -454.7192367601246,
|
||
|
|
"logps/rejected": -385.7163989028213,
|
||
|
|
"loss": 0.5881,
|
||
|
|
"loss/base_kto": 3.765181064605713,
|
||
|
|
"metrics/advantage_var": 250.76968383789062,
|
||
|
|
"regularization/mmd": 0.9399226307868958,
|
||
|
|
"regularization/rkhs_norm": 181.10263061523438,
|
||
|
|
"rewards/chosen": 0.11088201932817976,
|
||
|
|
"rewards/margins": 0.2165571336877813,
|
||
|
|
"rewards/rejected": -0.10567511435960154,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 14.423428535461426,
|
||
|
|
"kl": 14.862100601196289,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -35039565.84326019,
|
||
|
|
"logits/rejected": -36648494.25545172,
|
||
|
|
"logps/chosen": -494.4254506269593,
|
||
|
|
"logps/rejected": -377.2711253894081,
|
||
|
|
"loss": 0.5786,
|
||
|
|
"loss/base_kto": 3.728292942047119,
|
||
|
|
"metrics/advantage_var": 232.7931671142578,
|
||
|
|
"regularization/mmd": 0.9003773927688599,
|
||
|
|
"regularization/rkhs_norm": 173.4831085205078,
|
||
|
|
"rewards/chosen": 0.25897384213055935,
|
||
|
|
"rewards/margins": 0.22975997244746302,
|
||
|
|
"rewards/rejected": 0.029213869683096343,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 14.714305877685547,
|
||
|
|
"kl": 10.654457092285156,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -35113917.88346457,
|
||
|
|
"logits/rejected": -36209268.68837209,
|
||
|
|
"logps/chosen": -488.20344488188977,
|
||
|
|
"logps/rejected": -345.6830910852713,
|
||
|
|
"loss": 0.5819,
|
||
|
|
"loss/base_kto": 3.7753243446350098,
|
||
|
|
"metrics/advantage_var": 226.36376953125,
|
||
|
|
"regularization/mmd": 0.8800527453422546,
|
||
|
|
"regularization/rkhs_norm": 169.56700134277344,
|
||
|
|
"rewards/chosen": 0.08953093280942422,
|
||
|
|
"rewards/margins": 0.18774708573986876,
|
||
|
|
"rewards/rejected": -0.09821615293044453,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 16.04747772216797,
|
||
|
|
"kl": 10.848597526550293,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -36346454.25656878,
|
||
|
|
"logits/rejected": -37313227.02053712,
|
||
|
|
"logps/chosen": -500.8554385625966,
|
||
|
|
"logps/rejected": -379.5542061611374,
|
||
|
|
"loss": 0.5857,
|
||
|
|
"loss/base_kto": 3.7539193630218506,
|
||
|
|
"metrics/advantage_var": 289.1528015136719,
|
||
|
|
"regularization/mmd": 0.9315635561943054,
|
||
|
|
"regularization/rkhs_norm": 179.4920196533203,
|
||
|
|
"rewards/chosen": 0.12201917079353627,
|
||
|
|
"rewards/margins": 0.21513198979623918,
|
||
|
|
"rewards/rejected": -0.0931128190027029,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 17.332683563232422,
|
||
|
|
"kl": 12.433494567871094,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36929741.46278317,
|
||
|
|
"logits/rejected": -37516421.027190335,
|
||
|
|
"logps/chosen": -492.72314927184465,
|
||
|
|
"logps/rejected": -386.1748017371601,
|
||
|
|
"loss": 0.5809,
|
||
|
|
"loss/base_kto": 3.727473020553589,
|
||
|
|
"metrics/advantage_var": 243.08250427246094,
|
||
|
|
"regularization/mmd": 0.9198707938194275,
|
||
|
|
"regularization/rkhs_norm": 177.2390594482422,
|
||
|
|
"rewards/chosen": 0.15290815236113217,
|
||
|
|
"rewards/margins": 0.2302717687513028,
|
||
|
|
"rewards/rejected": -0.07736361639017064,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 14.800250053405762,
|
||
|
|
"kl": 8.272868156433105,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -35740746.76825397,
|
||
|
|
"logits/rejected": -37316790.74461538,
|
||
|
|
"logps/chosen": -477.70992063492065,
|
||
|
|
"logps/rejected": -360.88326923076926,
|
||
|
|
"loss": 0.5816,
|
||
|
|
"loss/base_kto": 3.7273449897766113,
|
||
|
|
"metrics/advantage_var": 246.2641143798828,
|
||
|
|
"regularization/mmd": 0.9251508712768555,
|
||
|
|
"regularization/rkhs_norm": 178.2564239501953,
|
||
|
|
"rewards/chosen": 0.00815635635739281,
|
||
|
|
"rewards/margins": 0.2315292529920082,
|
||
|
|
"rewards/rejected": -0.22337289663461538,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 8.796030044555664,
|
||
|
|
"kl": 6.298320293426514,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -34560059.65048543,
|
||
|
|
"logits/rejected": -35027130.18181818,
|
||
|
|
"logps/chosen": -467.2739178802589,
|
||
|
|
"logps/rejected": -387.8612215909091,
|
||
|
|
"loss": 0.5722,
|
||
|
|
"loss/base_kto": 3.5254478454589844,
|
||
|
|
"metrics/advantage_var": 362.2047424316406,
|
||
|
|
"regularization/mmd": 1.0523730516433716,
|
||
|
|
"regularization/rkhs_norm": 202.76939392089844,
|
||
|
|
"rewards/chosen": 0.14243385784062754,
|
||
|
|
"rewards/margins": 0.4932251647697824,
|
||
|
|
"rewards/rejected": -0.35079130692915483,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 16.780546188354492,
|
||
|
|
"kl": 10.918546676635742,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -35739378.27871363,
|
||
|
|
"logits/rejected": -36621637.00159489,
|
||
|
|
"logps/chosen": -463.666586906585,
|
||
|
|
"logps/rejected": -387.4616726475279,
|
||
|
|
"loss": 0.5696,
|
||
|
|
"loss/base_kto": 3.2663350105285645,
|
||
|
|
"metrics/advantage_var": 559.0786743164062,
|
||
|
|
"regularization/mmd": 1.2902926206588745,
|
||
|
|
"regularization/rkhs_norm": 248.61131286621094,
|
||
|
|
"rewards/chosen": 0.449957340818817,
|
||
|
|
"rewards/margins": 0.8422214073294205,
|
||
|
|
"rewards/rejected": -0.3922640665106036,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 13.12932014465332,
|
||
|
|
"kl": 18.999629974365234,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -34947555.018867925,
|
||
|
|
"logits/rejected": -36489925.16770186,
|
||
|
|
"logps/chosen": -442.7131977201258,
|
||
|
|
"logps/rejected": -391.82603843167703,
|
||
|
|
"loss": 0.5645,
|
||
|
|
"loss/base_kto": 3.246511220932007,
|
||
|
|
"metrics/advantage_var": 518.4884643554688,
|
||
|
|
"regularization/mmd": 1.2694826126098633,
|
||
|
|
"regularization/rkhs_norm": 244.60166931152344,
|
||
|
|
"rewards/chosen": 0.5303995384360259,
|
||
|
|
"rewards/margins": 0.7891794314463254,
|
||
|
|
"rewards/rejected": -0.2587798930102994,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 9.967662811279297,
|
||
|
|
"kl": 6.526576519012451,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -35417250.03588907,
|
||
|
|
"logits/rejected": -35428815.64017991,
|
||
|
|
"logps/chosen": -511.6195962479608,
|
||
|
|
"logps/rejected": -366.17344452773614,
|
||
|
|
"loss": 0.5673,
|
||
|
|
"loss/base_kto": 3.1759274005889893,
|
||
|
|
"metrics/advantage_var": 602.5060424804688,
|
||
|
|
"regularization/mmd": 1.3626421689987183,
|
||
|
|
"regularization/rkhs_norm": 262.5514831542969,
|
||
|
|
"rewards/chosen": 0.3481656894403803,
|
||
|
|
"rewards/margins": 0.9205143112720425,
|
||
|
|
"rewards/rejected": -0.5723486218316622,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 11.86622142791748,
|
||
|
|
"kl": 8.370628356933594,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -36123243.83151326,
|
||
|
|
"logits/rejected": -36291444.58215962,
|
||
|
|
"logps/chosen": -461.5860471918877,
|
||
|
|
"logps/rejected": -374.5848982785603,
|
||
|
|
"loss": 0.5643,
|
||
|
|
"loss/base_kto": 3.240628957748413,
|
||
|
|
"metrics/advantage_var": 543.981689453125,
|
||
|
|
"regularization/mmd": 1.2740906476974487,
|
||
|
|
"regularization/rkhs_norm": 245.48953247070312,
|
||
|
|
"rewards/chosen": 0.3063705849015211,
|
||
|
|
"rewards/margins": 0.820691638992997,
|
||
|
|
"rewards/rejected": -0.514321054091476,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 10.228275299072266,
|
||
|
|
"kl": 20.769678115844727,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -35856915.54198473,
|
||
|
|
"logits/rejected": -36818753.9456,
|
||
|
|
"logps/chosen": -459.0350190839695,
|
||
|
|
"logps/rejected": -369.253525,
|
||
|
|
"loss": 0.5539,
|
||
|
|
"loss/base_kto": 3.094128370285034,
|
||
|
|
"metrics/advantage_var": 562.14501953125,
|
||
|
|
"regularization/mmd": 1.3369877338409424,
|
||
|
|
"regularization/rkhs_norm": 257.6084289550781,
|
||
|
|
"rewards/chosen": 0.6964062872733778,
|
||
|
|
"rewards/margins": 0.9727445929374403,
|
||
|
|
"rewards/rejected": -0.2763383056640625,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 12.683265686035156,
|
||
|
|
"kl": 13.78448486328125,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -37744494.59818731,
|
||
|
|
"logits/rejected": -37460488.284789644,
|
||
|
|
"logps/chosen": -507.40563632930514,
|
||
|
|
"logps/rejected": -383.04634405339806,
|
||
|
|
"loss": 0.5758,
|
||
|
|
"loss/base_kto": 3.207179307937622,
|
||
|
|
"metrics/advantage_var": 653.6361694335938,
|
||
|
|
"regularization/mmd": 1.3994256258010864,
|
||
|
|
"regularization/rkhs_norm": 269.63885498046875,
|
||
|
|
"rewards/chosen": 0.5503196140070337,
|
||
|
|
"rewards/margins": 0.8896718235982048,
|
||
|
|
"rewards/rejected": -0.33935220959117113,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 11.519844055175781,
|
||
|
|
"kl": 10.403990745544434,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -35192848.84210526,
|
||
|
|
"logits/rejected": -35591631.23809524,
|
||
|
|
"logps/chosen": -488.888671875,
|
||
|
|
"logps/rejected": -376.9961867559524,
|
||
|
|
"loss": 0.5679,
|
||
|
|
"loss/base_kto": 3.162940263748169,
|
||
|
|
"metrics/advantage_var": 678.7843627929688,
|
||
|
|
"regularization/mmd": 1.3800307512283325,
|
||
|
|
"regularization/rkhs_norm": 265.9018859863281,
|
||
|
|
"rewards/chosen": 0.44200319992868525,
|
||
|
|
"rewards/margins": 0.9566270689617722,
|
||
|
|
"rewards/rejected": -0.5146238690330869,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 14.86784839630127,
|
||
|
|
"kl": 12.996451377868652,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -36329521.23076923,
|
||
|
|
"logits/rejected": -37399982.82926829,
|
||
|
|
"logps/chosen": -488.44190705128204,
|
||
|
|
"logps/rejected": -372.8462271341463,
|
||
|
|
"loss": 0.5731,
|
||
|
|
"loss/base_kto": 3.1849658489227295,
|
||
|
|
"metrics/advantage_var": 616.439453125,
|
||
|
|
"regularization/mmd": 1.400101900100708,
|
||
|
|
"regularization/rkhs_norm": 269.7691345214844,
|
||
|
|
"rewards/chosen": 0.5117607116699219,
|
||
|
|
"rewards/margins": 0.909477513010909,
|
||
|
|
"rewards/rejected": -0.397716801340987,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 12.045408248901367,
|
||
|
|
"kl": 11.645041465759277,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -36871137.835658915,
|
||
|
|
"logits/rejected": -38185995.28818898,
|
||
|
|
"logps/chosen": -492.06763565891475,
|
||
|
|
"logps/rejected": -380.04018208661415,
|
||
|
|
"loss": 0.5669,
|
||
|
|
"loss/base_kto": 3.208451986312866,
|
||
|
|
"metrics/advantage_var": 539.9119873046875,
|
||
|
|
"regularization/mmd": 1.3271362781524658,
|
||
|
|
"regularization/rkhs_norm": 255.7102508544922,
|
||
|
|
"rewards/chosen": 0.45313749091569766,
|
||
|
|
"rewards/margins": 0.8564526610836356,
|
||
|
|
"rewards/rejected": -0.403315170167938,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 13.22069263458252,
|
||
|
|
"kl": 12.721824645996094,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -35281307.51401869,
|
||
|
|
"logits/rejected": -36470151.623824455,
|
||
|
|
"logps/chosen": -474.9979556074766,
|
||
|
|
"logps/rejected": -394.079006661442,
|
||
|
|
"loss": 0.5703,
|
||
|
|
"loss/base_kto": 3.217174530029297,
|
||
|
|
"metrics/advantage_var": 539.5971069335938,
|
||
|
|
"regularization/mmd": 1.3451206684112549,
|
||
|
|
"regularization/rkhs_norm": 259.17547607421875,
|
||
|
|
"rewards/chosen": 0.49599230995059385,
|
||
|
|
"rewards/margins": 0.8761570273826937,
|
||
|
|
"rewards/rejected": -0.38016471743209984,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 13.768370628356934,
|
||
|
|
"kl": 14.06915283203125,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34588707.5892575,
|
||
|
|
"logits/rejected": -34524313.52086553,
|
||
|
|
"logps/chosen": -470.2716725908373,
|
||
|
|
"logps/rejected": -405.7633307573416,
|
||
|
|
"loss": 0.5818,
|
||
|
|
"loss/base_kto": 3.2592437267303467,
|
||
|
|
"metrics/advantage_var": 650.3941040039062,
|
||
|
|
"regularization/mmd": 1.3952311277389526,
|
||
|
|
"regularization/rkhs_norm": 268.8306579589844,
|
||
|
|
"rewards/chosen": 0.43066994424491506,
|
||
|
|
"rewards/margins": 0.8314947018400658,
|
||
|
|
"rewards/rejected": -0.4008247575951507,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 15.858077049255371,
|
||
|
|
"kl": 17.79615592956543,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -35441875.54491018,
|
||
|
|
"logits/rejected": -35059216.732026145,
|
||
|
|
"logps/chosen": -476.67215568862275,
|
||
|
|
"logps/rejected": -385.0362796160131,
|
||
|
|
"loss": 0.5624,
|
||
|
|
"loss/base_kto": 3.196995496749878,
|
||
|
|
"metrics/advantage_var": 541.07373046875,
|
||
|
|
"regularization/mmd": 1.3020113706588745,
|
||
|
|
"regularization/rkhs_norm": 250.86924743652344,
|
||
|
|
"rewards/chosen": 0.5837789295676226,
|
||
|
|
"rewards/margins": 0.8488349412911596,
|
||
|
|
"rewards/rejected": -0.26505601172353704,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 14.126097679138184,
|
||
|
|
"kl": 20.838333129882812,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -34667620.09022556,
|
||
|
|
"logits/rejected": -36130579.56422764,
|
||
|
|
"logps/chosen": -509.91494360902254,
|
||
|
|
"logps/rejected": -364.2094004065041,
|
||
|
|
"loss": 0.5678,
|
||
|
|
"loss/base_kto": 3.2881786823272705,
|
||
|
|
"metrics/advantage_var": 461.0966796875,
|
||
|
|
"regularization/mmd": 1.2540638446807861,
|
||
|
|
"regularization/rkhs_norm": 241.6307830810547,
|
||
|
|
"rewards/chosen": 0.5731052943638393,
|
||
|
|
"rewards/margins": 0.7670012204904589,
|
||
|
|
"rewards/rejected": -0.19389592612661966,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 13.985225677490234,
|
||
|
|
"kl": 16.84873390197754,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -35178023.133757964,
|
||
|
|
"logits/rejected": -35479306.99386503,
|
||
|
|
"logps/chosen": -491.51462977707007,
|
||
|
|
"logps/rejected": -377.34854294478527,
|
||
|
|
"loss": 0.5766,
|
||
|
|
"loss/base_kto": 3.1906678676605225,
|
||
|
|
"metrics/advantage_var": 649.16259765625,
|
||
|
|
"regularization/mmd": 1.4221806526184082,
|
||
|
|
"regularization/rkhs_norm": 274.0232849121094,
|
||
|
|
"rewards/chosen": 0.6344547636190038,
|
||
|
|
"rewards/margins": 0.9031659941251986,
|
||
|
|
"rewards/rejected": -0.2687112305061949,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 13.469324111938477,
|
||
|
|
"kl": 10.265970230102539,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -36038125.278219394,
|
||
|
|
"logits/rejected": -37385028.817204304,
|
||
|
|
"logps/chosen": -503.97322138314786,
|
||
|
|
"logps/rejected": -367.8606950844854,
|
||
|
|
"loss": 0.5608,
|
||
|
|
"loss/base_kto": 3.1485276222229004,
|
||
|
|
"metrics/advantage_var": 526.8321533203125,
|
||
|
|
"regularization/mmd": 1.3380202054977417,
|
||
|
|
"regularization/rkhs_norm": 257.8074035644531,
|
||
|
|
"rewards/chosen": 0.47844770034280604,
|
||
|
|
"rewards/margins": 0.9144246341165676,
|
||
|
|
"rewards/rejected": -0.4359769337737615,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 9.046934127807617,
|
||
|
|
"kl": 16.51395034790039,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34729330.04361371,
|
||
|
|
"logits/rejected": -36002616.97805642,
|
||
|
|
"logps/chosen": -469.17416277258565,
|
||
|
|
"logps/rejected": -377.32381465517244,
|
||
|
|
"loss": 0.5561,
|
||
|
|
"loss/base_kto": 3.212134599685669,
|
||
|
|
"metrics/advantage_var": 491.69171142578125,
|
||
|
|
"regularization/mmd": 1.2363241910934448,
|
||
|
|
"regularization/rkhs_norm": 238.21275329589844,
|
||
|
|
"rewards/chosen": 0.4971886750693633,
|
||
|
|
"rewards/margins": 0.8227839687422179,
|
||
|
|
"rewards/rejected": -0.3255952936728546,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 13.699871063232422,
|
||
|
|
"kl": 12.540992736816406,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -35314302.55250404,
|
||
|
|
"logits/rejected": -36094472.5204236,
|
||
|
|
"logps/chosen": -469.674071082391,
|
||
|
|
"logps/rejected": -378.0487660741301,
|
||
|
|
"loss": 0.56,
|
||
|
|
"loss/base_kto": 3.1778485774993896,
|
||
|
|
"metrics/advantage_var": 510.32958984375,
|
||
|
|
"regularization/mmd": 1.3021544218063354,
|
||
|
|
"regularization/rkhs_norm": 250.8968048095703,
|
||
|
|
"rewards/chosen": 0.4937756958993841,
|
||
|
|
"rewards/margins": 0.8691603512732848,
|
||
|
|
"rewards/rejected": -0.3753846553739008,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 13.748052597045898,
|
||
|
|
"kl": 7.949827671051025,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -34263613.04928458,
|
||
|
|
"logits/rejected": -35533221.55453149,
|
||
|
|
"logps/chosen": -486.01952503974564,
|
||
|
|
"logps/rejected": -376.6563220046083,
|
||
|
|
"loss": 0.5637,
|
||
|
|
"loss/base_kto": 3.226621389389038,
|
||
|
|
"metrics/advantage_var": 541.7657470703125,
|
||
|
|
"regularization/mmd": 1.2831403017044067,
|
||
|
|
"regularization/rkhs_norm": 247.2332000732422,
|
||
|
|
"rewards/chosen": 0.3904226330391743,
|
||
|
|
"rewards/margins": 0.8421150882313067,
|
||
|
|
"rewards/rejected": -0.4516924551921323,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 16.558958053588867,
|
||
|
|
"kl": 7.912838935852051,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -34892940.613893375,
|
||
|
|
"logits/rejected": -36078274.42057489,
|
||
|
|
"logps/chosen": -483.1467588852989,
|
||
|
|
"logps/rejected": -374.8048411497731,
|
||
|
|
"loss": 0.5645,
|
||
|
|
"loss/base_kto": 3.155876636505127,
|
||
|
|
"metrics/advantage_var": 615.82958984375,
|
||
|
|
"regularization/mmd": 1.3601619005203247,
|
||
|
|
"regularization/rkhs_norm": 262.0735778808594,
|
||
|
|
"rewards/chosen": 0.43474848782688813,
|
||
|
|
"rewards/margins": 0.9497797570331098,
|
||
|
|
"rewards/rejected": -0.5150312692062217,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 13.430770874023438,
|
||
|
|
"kl": 13.892699241638184,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34218263.87138264,
|
||
|
|
"logits/rejected": -35506754.91793313,
|
||
|
|
"logps/chosen": -497.61223874598073,
|
||
|
|
"logps/rejected": -345.22314304711244,
|
||
|
|
"loss": 0.5528,
|
||
|
|
"loss/base_kto": 3.1559383869171143,
|
||
|
|
"metrics/advantage_var": 486.2275390625,
|
||
|
|
"regularization/mmd": 1.2661854028701782,
|
||
|
|
"regularization/rkhs_norm": 243.9663543701172,
|
||
|
|
"rewards/chosen": 0.552090512977919,
|
||
|
|
"rewards/margins": 0.8996528912786409,
|
||
|
|
"rewards/rejected": -0.3475623783007219,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 15.6405668258667,
|
||
|
|
"kl": 14.188281059265137,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -34322806.30014859,
|
||
|
|
"logits/rejected": -37109854.47116969,
|
||
|
|
"logps/chosen": -503.85944465081724,
|
||
|
|
"logps/rejected": -364.56785420098845,
|
||
|
|
"loss": 0.5777,
|
||
|
|
"loss/base_kto": 3.1833126544952393,
|
||
|
|
"metrics/advantage_var": 741.0894165039062,
|
||
|
|
"regularization/mmd": 1.4379055500030518,
|
||
|
|
"regularization/rkhs_norm": 277.0531005859375,
|
||
|
|
"rewards/chosen": 0.5156612311217265,
|
||
|
|
"rewards/margins": 0.8787353688266919,
|
||
|
|
"rewards/rejected": -0.3630741377049655,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 28.624948501586914,
|
||
|
|
"kl": 12.644123077392578,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -34743139.68847352,
|
||
|
|
"logits/rejected": -36302250.93416928,
|
||
|
|
"logps/chosen": -463.32496105919006,
|
||
|
|
"logps/rejected": -376.3301332288401,
|
||
|
|
"loss": 0.5646,
|
||
|
|
"loss/base_kto": 3.245013475418091,
|
||
|
|
"metrics/advantage_var": 545.5440673828125,
|
||
|
|
"regularization/mmd": 1.271781325340271,
|
||
|
|
"regularization/rkhs_norm": 245.04458618164062,
|
||
|
|
"rewards/chosen": 0.5009703829281056,
|
||
|
|
"rewards/margins": 0.8233771158520378,
|
||
|
|
"rewards/rejected": -0.32240673292393224,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 11.098860740661621,
|
||
|
|
"kl": 13.221571922302246,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -34350759.41587301,
|
||
|
|
"logits/rejected": -36409989.907692306,
|
||
|
|
"logps/chosen": -507.50555555555553,
|
||
|
|
"logps/rejected": -376.8789903846154,
|
||
|
|
"loss": 0.5695,
|
||
|
|
"loss/base_kto": 3.155639171600342,
|
||
|
|
"metrics/advantage_var": 640.3905029296875,
|
||
|
|
"regularization/mmd": 1.4001741409301758,
|
||
|
|
"regularization/rkhs_norm": 269.7830810546875,
|
||
|
|
"rewards/chosen": 0.5849505227709574,
|
||
|
|
"rewards/margins": 0.9541778881180127,
|
||
|
|
"rewards/rejected": -0.3692273653470553,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 13.457465171813965,
|
||
|
|
"kl": 13.062397956848145,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -34188224.50874404,
|
||
|
|
"logits/rejected": -35642448.22119816,
|
||
|
|
"logps/chosen": -482.57839825119237,
|
||
|
|
"logps/rejected": -355.75254416282644,
|
||
|
|
"loss": 0.5626,
|
||
|
|
"loss/base_kto": 3.2334468364715576,
|
||
|
|
"metrics/advantage_var": 501.9878845214844,
|
||
|
|
"regularization/mmd": 1.2672839164733887,
|
||
|
|
"regularization/rkhs_norm": 244.1780242919922,
|
||
|
|
"rewards/chosen": 0.4810851662640352,
|
||
|
|
"rewards/margins": 0.8089145067168242,
|
||
|
|
"rewards/rejected": -0.327829340452789,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 9.959589958190918,
|
||
|
|
"kl": 10.90673828125,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -35522215.98744113,
|
||
|
|
"logits/rejected": -36263321.281493,
|
||
|
|
"logps/chosen": -472.88196624803766,
|
||
|
|
"logps/rejected": -369.7211314152411,
|
||
|
|
"loss": 0.5723,
|
||
|
|
"loss/base_kto": 3.2004776000976562,
|
||
|
|
"metrics/advantage_var": 601.2073364257812,
|
||
|
|
"regularization/mmd": 1.3777668476104736,
|
||
|
|
"regularization/rkhs_norm": 265.4656677246094,
|
||
|
|
"rewards/chosen": 0.44117609859448587,
|
||
|
|
"rewards/margins": 0.8610656999504345,
|
||
|
|
"rewards/rejected": -0.41988960135594866,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 10.555566787719727,
|
||
|
|
"kl": 14.242239952087402,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -33768006.42967542,
|
||
|
|
"logits/rejected": -35118029.85150079,
|
||
|
|
"logps/chosen": -469.5661707882535,
|
||
|
|
"logps/rejected": -368.74012638230647,
|
||
|
|
"loss": 0.5527,
|
||
|
|
"loss/base_kto": 3.1599979400634766,
|
||
|
|
"metrics/advantage_var": 506.82470703125,
|
||
|
|
"regularization/mmd": 1.2614637613296509,
|
||
|
|
"regularization/rkhs_norm": 243.0565948486328,
|
||
|
|
"rewards/chosen": 0.531270942511109,
|
||
|
|
"rewards/margins": 0.8937158758796171,
|
||
|
|
"rewards/rejected": -0.3624449333685081,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 11.619686126708984,
|
||
|
|
"kl": 9.854671478271484,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -35102347.63636363,
|
||
|
|
"logits/rejected": -35890413.4939759,
|
||
|
|
"logps/chosen": -484.1734983766234,
|
||
|
|
"logps/rejected": -387.3549510542169,
|
||
|
|
"loss": 0.5709,
|
||
|
|
"loss/base_kto": 3.2288010120391846,
|
||
|
|
"metrics/advantage_var": 594.1427612304688,
|
||
|
|
"regularization/mmd": 1.3385165929794312,
|
||
|
|
"regularization/rkhs_norm": 257.90301513671875,
|
||
|
|
"rewards/chosen": 0.46015707238928066,
|
||
|
|
"rewards/margins": 0.8614431401221261,
|
||
|
|
"rewards/rejected": -0.4012860677328454,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 15.014073371887207,
|
||
|
|
"kl": 9.039198875427246,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -35080624.8085758,
|
||
|
|
"logits/rejected": -36249443.215311006,
|
||
|
|
"logps/chosen": -483.1822358346095,
|
||
|
|
"logps/rejected": -375.1662679425837,
|
||
|
|
"loss": 0.5656,
|
||
|
|
"loss/base_kto": 3.224367618560791,
|
||
|
|
"metrics/advantage_var": 550.4187622070312,
|
||
|
|
"regularization/mmd": 1.300495982170105,
|
||
|
|
"regularization/rkhs_norm": 250.57725524902344,
|
||
|
|
"rewards/chosen": 0.4142883674656633,
|
||
|
|
"rewards/margins": 0.865131606600432,
|
||
|
|
"rewards/rejected": -0.45084323913476876,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 11.960787773132324,
|
||
|
|
"kl": 13.760507583618164,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -34482436.711656444,
|
||
|
|
"logits/rejected": -35953145.477707006,
|
||
|
|
"logps/chosen": -483.8002300613497,
|
||
|
|
"logps/rejected": -383.39570063694265,
|
||
|
|
"loss": 0.5548,
|
||
|
|
"loss/base_kto": 3.189375638961792,
|
||
|
|
"metrics/advantage_var": 481.8376159667969,
|
||
|
|
"regularization/mmd": 1.249145269393921,
|
||
|
|
"regularization/rkhs_norm": 240.68309020996094,
|
||
|
|
"rewards/chosen": 0.49734628127396474,
|
||
|
|
"rewards/margins": 0.8299500031355914,
|
||
|
|
"rewards/rejected": -0.3326037218616267,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 15.612455368041992,
|
||
|
|
"kl": 18.639978408813477,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -35105800.03767661,
|
||
|
|
"logits/rejected": -35919365.57387247,
|
||
|
|
"logps/chosen": -473.2154140502355,
|
||
|
|
"logps/rejected": -392.6224241835148,
|
||
|
|
"loss": 0.5577,
|
||
|
|
"loss/base_kto": 3.197514057159424,
|
||
|
|
"metrics/advantage_var": 516.6060791015625,
|
||
|
|
"regularization/mmd": 1.2640329599380493,
|
||
|
|
"regularization/rkhs_norm": 243.5516357421875,
|
||
|
|
"rewards/chosen": 0.5329540019132653,
|
||
|
|
"rewards/margins": 0.8422518394272378,
|
||
|
|
"rewards/rejected": -0.3092978375139726,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 13.539543151855469,
|
||
|
|
"kl": 12.182269096374512,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -35346172.52488688,
|
||
|
|
"logits/rejected": -35774855.67585089,
|
||
|
|
"logps/chosen": -473.17444381598796,
|
||
|
|
"logps/rejected": -350.419545178282,
|
||
|
|
"loss": 0.5706,
|
||
|
|
"loss/base_kto": 3.2329118251800537,
|
||
|
|
"metrics/advantage_var": 542.537109375,
|
||
|
|
"regularization/mmd": 1.3316736221313477,
|
||
|
|
"regularization/rkhs_norm": 256.58453369140625,
|
||
|
|
"rewards/chosen": 0.5181275334653092,
|
||
|
|
"rewards/margins": 0.8476684600600974,
|
||
|
|
"rewards/rejected": -0.3295409265947883,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 10.709891319274902,
|
||
|
|
"kl": 9.94327449798584,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -35305236.82296651,
|
||
|
|
"logits/rejected": -35859495.203675345,
|
||
|
|
"logps/chosen": -484.4208034290271,
|
||
|
|
"logps/rejected": -359.55044027565083,
|
||
|
|
"loss": 0.5594,
|
||
|
|
"loss/base_kto": 3.2328438758850098,
|
||
|
|
"metrics/advantage_var": 489.0275573730469,
|
||
|
|
"regularization/mmd": 1.242189645767212,
|
||
|
|
"regularization/rkhs_norm": 239.3428955078125,
|
||
|
|
"rewards/chosen": 0.4519747257993172,
|
||
|
|
"rewards/margins": 0.8191801250832711,
|
||
|
|
"rewards/rejected": -0.36720539928395385,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 13.026509284973145,
|
||
|
|
"kl": 14.385235786437988,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -33401099.403993856,
|
||
|
|
"logits/rejected": -34391699.33227345,
|
||
|
|
"logps/chosen": -475.46548579109066,
|
||
|
|
"logps/rejected": -352.3910472972973,
|
||
|
|
"loss": 0.5597,
|
||
|
|
"loss/base_kto": 3.1858298778533936,
|
||
|
|
"metrics/advantage_var": 524.2139892578125,
|
||
|
|
"regularization/mmd": 1.2919800281524658,
|
||
|
|
"regularization/rkhs_norm": 248.93643188476562,
|
||
|
|
"rewards/chosen": 0.5690245738227246,
|
||
|
|
"rewards/margins": 0.8721156503082175,
|
||
|
|
"rewards/rejected": -0.3030910764854928,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 9.928730010986328,
|
||
|
|
"kl": 13.804104804992676,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34858126.6625387,
|
||
|
|
"logits/rejected": -35866927.646687694,
|
||
|
|
"logps/chosen": -468.6269349845201,
|
||
|
|
"logps/rejected": -394.26695583596216,
|
||
|
|
"loss": 0.5585,
|
||
|
|
"loss/base_kto": 3.2086141109466553,
|
||
|
|
"metrics/advantage_var": 477.46563720703125,
|
||
|
|
"regularization/mmd": 1.2596110105514526,
|
||
|
|
"regularization/rkhs_norm": 242.69961547851562,
|
||
|
|
"rewards/chosen": 0.5428633084607198,
|
||
|
|
"rewards/margins": 0.849956281947202,
|
||
|
|
"rewards/rejected": -0.3070929734864822,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 15.070842742919922,
|
||
|
|
"kl": 11.10472297668457,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -34750306.461538464,
|
||
|
|
"logits/rejected": -35542640.15238095,
|
||
|
|
"logps/chosen": -494.4030769230769,
|
||
|
|
"logps/rejected": -391.96760912698414,
|
||
|
|
"loss": 0.5614,
|
||
|
|
"loss/base_kto": 3.1633832454681396,
|
||
|
|
"metrics/advantage_var": 531.6734619140625,
|
||
|
|
"regularization/mmd": 1.3275425434112549,
|
||
|
|
"regularization/rkhs_norm": 255.78855895996094,
|
||
|
|
"rewards/chosen": 0.5454000150240385,
|
||
|
|
"rewards/margins": 0.928437255188659,
|
||
|
|
"rewards/rejected": -0.38303724016462054,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 14.644248008728027,
|
||
|
|
"kl": 13.015934944152832,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -35368389.299363054,
|
||
|
|
"logits/rejected": -37573644.564417176,
|
||
|
|
"logps/chosen": -474.7845342356688,
|
||
|
|
"logps/rejected": -372.0687787576687,
|
||
|
|
"loss": 0.5603,
|
||
|
|
"loss/base_kto": 3.153552293777466,
|
||
|
|
"metrics/advantage_var": 546.5127563476562,
|
||
|
|
"regularization/mmd": 1.3292239904403687,
|
||
|
|
"regularization/rkhs_norm": 256.1125183105469,
|
||
|
|
"rewards/chosen": 0.5453437270632215,
|
||
|
|
"rewards/margins": 0.9080075577794073,
|
||
|
|
"rewards/rejected": -0.3626638307161858,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 13.781595230102539,
|
||
|
|
"kl": 9.892565727233887,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -34171076.46511628,
|
||
|
|
"logits/rejected": -35169127.783783786,
|
||
|
|
"logps/chosen": -478.1899527616279,
|
||
|
|
"logps/rejected": -376.4047719594595,
|
||
|
|
"loss": 0.565,
|
||
|
|
"loss/base_kto": 3.2333877086639404,
|
||
|
|
"metrics/advantage_var": 520.9701538085938,
|
||
|
|
"regularization/mmd": 1.2864378690719604,
|
||
|
|
"regularization/rkhs_norm": 247.8685760498047,
|
||
|
|
"rewards/chosen": 0.49371550803960756,
|
||
|
|
"rewards/margins": 0.8383285575209737,
|
||
|
|
"rewards/rejected": -0.3446130494813661,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 14.64671802520752,
|
||
|
|
"kl": 12.811894416809082,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -36281189.3418124,
|
||
|
|
"logits/rejected": -36737571.391705066,
|
||
|
|
"logps/chosen": -486.2001689189189,
|
||
|
|
"logps/rejected": -389.9088421658986,
|
||
|
|
"loss": 0.5675,
|
||
|
|
"loss/base_kto": 3.20735239982605,
|
||
|
|
"metrics/advantage_var": 596.3021240234375,
|
||
|
|
"regularization/mmd": 1.3325361013412476,
|
||
|
|
"regularization/rkhs_norm": 256.7507019042969,
|
||
|
|
"rewards/chosen": 0.5178320176650686,
|
||
|
|
"rewards/margins": 0.8580788950277198,
|
||
|
|
"rewards/rejected": -0.3402468773626512,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 10.71275520324707,
|
||
|
|
"kl": 11.482906341552734,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34089391.686274506,
|
||
|
|
"logits/rejected": -35108587.24324324,
|
||
|
|
"logps/chosen": -487.71920955882354,
|
||
|
|
"logps/rejected": -377.3284769144144,
|
||
|
|
"loss": 0.5351,
|
||
|
|
"loss/base_kto": 3.1168744564056396,
|
||
|
|
"metrics/advantage_var": 399.5955505371094,
|
||
|
|
"regularization/mmd": 1.1641170978546143,
|
||
|
|
"regularization/rkhs_norm": 224.3000030517578,
|
||
|
|
"rewards/chosen": 0.5034692303027982,
|
||
|
|
"rewards/margins": 0.9092341566060547,
|
||
|
|
"rewards/rejected": -0.4057649263032564,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 11.445816040039062,
|
||
|
|
"kl": 10.277003288269043,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34275630.51107828,
|
||
|
|
"logits/rejected": -34560315.86069652,
|
||
|
|
"logps/chosen": -459.00415435745936,
|
||
|
|
"logps/rejected": -382.97128938640134,
|
||
|
|
"loss": 0.5324,
|
||
|
|
"loss/base_kto": 3.1620447635650635,
|
||
|
|
"metrics/advantage_var": 355.1131286621094,
|
||
|
|
"regularization/mmd": 1.0971364974975586,
|
||
|
|
"regularization/rkhs_norm": 211.3943328857422,
|
||
|
|
"rewards/chosen": 0.47600524992499077,
|
||
|
|
"rewards/margins": 0.8982038982037791,
|
||
|
|
"rewards/rejected": -0.42219864827878834,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 10.892109870910645,
|
||
|
|
"kl": 17.682931900024414,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -33588075.87421384,
|
||
|
|
"logits/rejected": -36002466.18633541,
|
||
|
|
"logps/chosen": -497.57596305031444,
|
||
|
|
"logps/rejected": -361.7705017468944,
|
||
|
|
"loss": 0.5269,
|
||
|
|
"loss/base_kto": 3.0482025146484375,
|
||
|
|
"metrics/advantage_var": 400.92138671875,
|
||
|
|
"regularization/mmd": 1.1668514013290405,
|
||
|
|
"regularization/rkhs_norm": 224.8268585205078,
|
||
|
|
"rewards/chosen": 0.7147384739521915,
|
||
|
|
"rewards/margins": 0.9691219639045159,
|
||
|
|
"rewards/rejected": -0.25438348995232435,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 11.56076717376709,
|
||
|
|
"kl": 13.388104438781738,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -34235619.381317,
|
||
|
|
"logits/rejected": -36256815.362041466,
|
||
|
|
"logps/chosen": -482.87964203675347,
|
||
|
|
"logps/rejected": -386.31733452950556,
|
||
|
|
"loss": 0.5271,
|
||
|
|
"loss/base_kto": 3.081200361251831,
|
||
|
|
"metrics/advantage_var": 383.1581115722656,
|
||
|
|
"regularization/mmd": 1.135278344154358,
|
||
|
|
"regularization/rkhs_norm": 218.743408203125,
|
||
|
|
"rewards/chosen": 0.6020886361325134,
|
||
|
|
"rewards/margins": 0.9377138785209329,
|
||
|
|
"rewards/rejected": -0.33562524238841956,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 9.227808952331543,
|
||
|
|
"kl": 13.091130256652832,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -34775515.83045526,
|
||
|
|
"logits/rejected": -34548301.23794712,
|
||
|
|
"logps/chosen": -482.2095270800628,
|
||
|
|
"logps/rejected": -374.9788102643857,
|
||
|
|
"loss": 0.5275,
|
||
|
|
"loss/base_kto": 3.0965659618377686,
|
||
|
|
"metrics/advantage_var": 363.8895568847656,
|
||
|
|
"regularization/mmd": 1.1230347156524658,
|
||
|
|
"regularization/rkhs_norm": 216.3843231201172,
|
||
|
|
"rewards/chosen": 0.5647210281348116,
|
||
|
|
"rewards/margins": 0.929788576871811,
|
||
|
|
"rewards/rejected": -0.3650675487369994,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 11.359593391418457,
|
||
|
|
"kl": 12.2516508102417,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -34294916.98701299,
|
||
|
|
"logits/rejected": -34889456.578313254,
|
||
|
|
"logps/chosen": -472.3590198863636,
|
||
|
|
"logps/rejected": -371.5574642319277,
|
||
|
|
"loss": 0.5298,
|
||
|
|
"loss/base_kto": 3.090224504470825,
|
||
|
|
"metrics/advantage_var": 379.31390380859375,
|
||
|
|
"regularization/mmd": 1.148317575454712,
|
||
|
|
"regularization/rkhs_norm": 221.2557830810547,
|
||
|
|
"rewards/chosen": 0.5197414051402699,
|
||
|
|
"rewards/margins": 0.9446700932815066,
|
||
|
|
"rewards/rejected": -0.4249286881412368,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 8.25497817993164,
|
||
|
|
"kl": 9.440899848937988,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -34211684.17391305,
|
||
|
|
"logits/rejected": -36721248.60377359,
|
||
|
|
"logps/chosen": -489.473262810559,
|
||
|
|
"logps/rejected": -362.4254373034591,
|
||
|
|
"loss": 0.5322,
|
||
|
|
"loss/base_kto": 3.1154391765594482,
|
||
|
|
"metrics/advantage_var": 390.202880859375,
|
||
|
|
"regularization/mmd": 1.14195716381073,
|
||
|
|
"regularization/rkhs_norm": 220.0302734375,
|
||
|
|
"rewards/chosen": 0.4888434084305852,
|
||
|
|
"rewards/margins": 0.9296738070861228,
|
||
|
|
"rewards/rejected": -0.4408303986555375,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 10.409468650817871,
|
||
|
|
"kl": 13.0885009765625,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34505938.58712716,
|
||
|
|
"logits/rejected": -36608060.516329706,
|
||
|
|
"logps/chosen": -523.9324960753532,
|
||
|
|
"logps/rejected": -372.30574455676515,
|
||
|
|
"loss": 0.5298,
|
||
|
|
"loss/base_kto": 3.0309512615203857,
|
||
|
|
"metrics/advantage_var": 430.7159729003906,
|
||
|
|
"regularization/mmd": 1.2071322202682495,
|
||
|
|
"regularization/rkhs_norm": 232.5880889892578,
|
||
|
|
"rewards/chosen": 0.6278587832368524,
|
||
|
|
"rewards/margins": 1.007938370043083,
|
||
|
|
"rewards/rejected": -0.38007958680623055,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 9.557205200195312,
|
||
|
|
"kl": 9.520075798034668,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -33973349.618320614,
|
||
|
|
"logits/rejected": -35741296.2304,
|
||
|
|
"logps/chosen": -473.8358778625954,
|
||
|
|
"logps/rejected": -392.514525,
|
||
|
|
"loss": 0.5327,
|
||
|
|
"loss/base_kto": 3.0503900051116943,
|
||
|
|
"metrics/advantage_var": 438.140625,
|
||
|
|
"regularization/mmd": 1.211198091506958,
|
||
|
|
"regularization/rkhs_norm": 233.37149047851562,
|
||
|
|
"rewards/chosen": 0.5725689371123569,
|
||
|
|
"rewards/margins": 1.0125099527373569,
|
||
|
|
"rewards/rejected": -0.439941015625,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 12.640892028808594,
|
||
|
|
"kl": 8.175743103027344,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -33956896.830534354,
|
||
|
|
"logits/rejected": -35551094.3744,
|
||
|
|
"logps/chosen": -492.9168893129771,
|
||
|
|
"logps/rejected": -365.5969,
|
||
|
|
"loss": 0.5307,
|
||
|
|
"loss/base_kto": 3.129976987838745,
|
||
|
|
"metrics/advantage_var": 352.7520446777344,
|
||
|
|
"regularization/mmd": 1.1156553030014038,
|
||
|
|
"regularization/rkhs_norm": 214.9624786376953,
|
||
|
|
"rewards/chosen": 0.5004034843153626,
|
||
|
|
"rewards/margins": 0.9101089530653627,
|
||
|
|
"rewards/rejected": -0.40970546875,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 8.18465518951416,
|
||
|
|
"kl": 9.268996238708496,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -34128800.34303216,
|
||
|
|
"logits/rejected": -34571063.119617224,
|
||
|
|
"logps/chosen": -443.77038667687594,
|
||
|
|
"logps/rejected": -373.73514752791067,
|
||
|
|
"loss": 0.5254,
|
||
|
|
"loss/base_kto": 3.1064136028289795,
|
||
|
|
"metrics/advantage_var": 385.292236328125,
|
||
|
|
"regularization/mmd": 1.0964959859848022,
|
||
|
|
"regularization/rkhs_norm": 211.27090454101562,
|
||
|
|
"rewards/chosen": 0.5316680861468702,
|
||
|
|
"rewards/margins": 0.9256872054339017,
|
||
|
|
"rewards/rejected": -0.3940191192870315,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 10.294926643371582,
|
||
|
|
"kl": 12.416064262390137,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35230866.741433024,
|
||
|
|
"logits/rejected": -35737593.5799373,
|
||
|
|
"logps/chosen": -487.93126947040497,
|
||
|
|
"logps/rejected": -393.0354868730407,
|
||
|
|
"loss": 0.5285,
|
||
|
|
"loss/base_kto": 3.10258150100708,
|
||
|
|
"metrics/advantage_var": 355.3718566894531,
|
||
|
|
"regularization/mmd": 1.125684142112732,
|
||
|
|
"regularization/rkhs_norm": 216.8948516845703,
|
||
|
|
"rewards/chosen": 0.548384431737977,
|
||
|
|
"rewards/margins": 0.9079973301310877,
|
||
|
|
"rewards/rejected": -0.3596128983931108,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 11.296473503112793,
|
||
|
|
"kl": 13.17083740234375,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -36390026.55054432,
|
||
|
|
"logits/rejected": -34921644.00627943,
|
||
|
|
"logps/chosen": -488.79840590979785,
|
||
|
|
"logps/rejected": -368.5661057692308,
|
||
|
|
"loss": 0.5294,
|
||
|
|
"loss/base_kto": 3.0887959003448486,
|
||
|
|
"metrics/advantage_var": 367.3639221191406,
|
||
|
|
"regularization/mmd": 1.1464333534240723,
|
||
|
|
"regularization/rkhs_norm": 220.8927459716797,
|
||
|
|
"rewards/chosen": 0.5507928305428654,
|
||
|
|
"rewards/margins": 0.9450203899184786,
|
||
|
|
"rewards/rejected": -0.3942275593756132,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 12.347671508789062,
|
||
|
|
"kl": 13.250162124633789,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -34845197.79397781,
|
||
|
|
"logits/rejected": -36435696.61633282,
|
||
|
|
"logps/chosen": -464.3315174326466,
|
||
|
|
"logps/rejected": -356.19414483821265,
|
||
|
|
"loss": 0.522,
|
||
|
|
"loss/base_kto": 3.090343952178955,
|
||
|
|
"metrics/advantage_var": 332.7373962402344,
|
||
|
|
"regularization/mmd": 1.0859272480010986,
|
||
|
|
"regularization/rkhs_norm": 209.23452758789062,
|
||
|
|
"rewards/chosen": 0.5313877401941363,
|
||
|
|
"rewards/margins": 0.9191312507181395,
|
||
|
|
"rewards/rejected": -0.3877435105240033,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 12.698050498962402,
|
||
|
|
"kl": 9.831679344177246,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -35244203.47551343,
|
||
|
|
"logits/rejected": -36054669.65069552,
|
||
|
|
"logps/chosen": -455.4088665086888,
|
||
|
|
"logps/rejected": -397.23357805255023,
|
||
|
|
"loss": 0.5281,
|
||
|
|
"loss/base_kto": 3.0877957344055176,
|
||
|
|
"metrics/advantage_var": 389.5352783203125,
|
||
|
|
"regularization/mmd": 1.136988639831543,
|
||
|
|
"regularization/rkhs_norm": 219.0729522705078,
|
||
|
|
"rewards/chosen": 0.4912326324607524,
|
||
|
|
"rewards/margins": 0.9503001171557388,
|
||
|
|
"rewards/rejected": -0.4590674846949865,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 11.359013557434082,
|
||
|
|
"kl": 8.866395950317383,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -35714928.671850696,
|
||
|
|
"logits/rejected": -35437547.10204082,
|
||
|
|
"logps/chosen": -471.7073289269051,
|
||
|
|
"logps/rejected": -394.2383241758242,
|
||
|
|
"loss": 0.5306,
|
||
|
|
"loss/base_kto": 3.0911548137664795,
|
||
|
|
"metrics/advantage_var": 391.9338073730469,
|
||
|
|
"regularization/mmd": 1.15353524684906,
|
||
|
|
"regularization/rkhs_norm": 222.2611083984375,
|
||
|
|
"rewards/chosen": 0.46505623397708495,
|
||
|
|
"rewards/margins": 0.9473504333951335,
|
||
|
|
"rewards/rejected": -0.48229419941804846,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 11.26817798614502,
|
||
|
|
"kl": 9.509455680847168,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -32140284.809968848,
|
||
|
|
"logits/rejected": -35454834.75862069,
|
||
|
|
"logps/chosen": -467.07028816199374,
|
||
|
|
"logps/rejected": -387.3153653996865,
|
||
|
|
"loss": 0.5266,
|
||
|
|
"loss/base_kto": 3.1238632202148438,
|
||
|
|
"metrics/advantage_var": 345.85382080078125,
|
||
|
|
"regularization/mmd": 1.08919358253479,
|
||
|
|
"regularization/rkhs_norm": 209.86387634277344,
|
||
|
|
"rewards/chosen": 0.48274901202905957,
|
||
|
|
"rewards/margins": 0.9038101780569299,
|
||
|
|
"rewards/rejected": -0.4210611660278703,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 11.294798851013184,
|
||
|
|
"kl": 14.53909969329834,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -34184551.96147673,
|
||
|
|
"logits/rejected": -35793104.852359205,
|
||
|
|
"logps/chosen": -474.4423154093098,
|
||
|
|
"logps/rejected": -384.1917332572298,
|
||
|
|
"loss": 0.5304,
|
||
|
|
"loss/base_kto": 3.1386189460754395,
|
||
|
|
"metrics/advantage_var": 370.9787902832031,
|
||
|
|
"regularization/mmd": 1.1045948266983032,
|
||
|
|
"regularization/rkhs_norm": 212.8313446044922,
|
||
|
|
"rewards/chosen": 0.5339062539187901,
|
||
|
|
"rewards/margins": 0.857013690286156,
|
||
|
|
"rewards/rejected": -0.3231074363673659,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 7.645028114318848,
|
||
|
|
"kl": 11.492669105529785,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -35577545.430615164,
|
||
|
|
"logits/rejected": -36086408.59208262,
|
||
|
|
"logps/chosen": -481.560443490701,
|
||
|
|
"logps/rejected": -389.5584391135973,
|
||
|
|
"loss": 0.5324,
|
||
|
|
"loss/base_kto": 3.1076765060424805,
|
||
|
|
"metrics/advantage_var": 388.7484436035156,
|
||
|
|
"regularization/mmd": 1.1512047052383423,
|
||
|
|
"regularization/rkhs_norm": 221.8120880126953,
|
||
|
|
"rewards/chosen": 0.5397454563299132,
|
||
|
|
"rewards/margins": 0.9362181180426122,
|
||
|
|
"rewards/rejected": -0.396472661712699,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 12.249076843261719,
|
||
|
|
"kl": 15.316606521606445,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -35357905.45454545,
|
||
|
|
"logits/rejected": -36790333.686746985,
|
||
|
|
"logps/chosen": -529.6217532467532,
|
||
|
|
"logps/rejected": -362.1625094126506,
|
||
|
|
"loss": 0.5283,
|
||
|
|
"loss/base_kto": 3.0658607482910156,
|
||
|
|
"metrics/advantage_var": 402.5772399902344,
|
||
|
|
"regularization/mmd": 1.1603775024414062,
|
||
|
|
"regularization/rkhs_norm": 223.57949829101562,
|
||
|
|
"rewards/chosen": 0.6278794523957488,
|
||
|
|
"rewards/margins": 0.9634614272066133,
|
||
|
|
"rewards/rejected": -0.33558197481086455,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 9.163156509399414,
|
||
|
|
"kl": 10.477073669433594,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -33207286.309148263,
|
||
|
|
"logits/rejected": -35143521.486068115,
|
||
|
|
"logps/chosen": -472.6181979495268,
|
||
|
|
"logps/rejected": -357.89870356037153,
|
||
|
|
"loss": 0.5287,
|
||
|
|
"loss/base_kto": 3.095052480697632,
|
||
|
|
"metrics/advantage_var": 384.4367980957031,
|
||
|
|
"regularization/mmd": 1.134222388267517,
|
||
|
|
"regularization/rkhs_norm": 218.5399932861328,
|
||
|
|
"rewards/chosen": 0.5564845713907236,
|
||
|
|
"rewards/margins": 0.927981380177972,
|
||
|
|
"rewards/rejected": -0.37149680878724844,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 10.682226181030273,
|
||
|
|
"kl": 11.542448997497559,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -35190588.49452269,
|
||
|
|
"logits/rejected": -34718846.20280811,
|
||
|
|
"logps/chosen": -460.3348493740219,
|
||
|
|
"logps/rejected": -384.1492297191888,
|
||
|
|
"loss": 0.5279,
|
||
|
|
"loss/base_kto": 3.1000070571899414,
|
||
|
|
"metrics/advantage_var": 357.6339416503906,
|
||
|
|
"regularization/mmd": 1.1230379343032837,
|
||
|
|
"regularization/rkhs_norm": 216.38499450683594,
|
||
|
|
"rewards/chosen": 0.5276913351855927,
|
||
|
|
"rewards/margins": 0.9278597198506668,
|
||
|
|
"rewards/rejected": -0.4001683846650741,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 14.015803337097168,
|
||
|
|
"kl": 12.815312385559082,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -35221306.30467571,
|
||
|
|
"logits/rejected": -36127556.460291736,
|
||
|
|
"logps/chosen": -473.0073058069382,
|
||
|
|
"logps/rejected": -369.95715153970826,
|
||
|
|
"loss": 0.5245,
|
||
|
|
"loss/base_kto": 3.0746517181396484,
|
||
|
|
"metrics/advantage_var": 359.9502868652344,
|
||
|
|
"regularization/mmd": 1.121084213256836,
|
||
|
|
"regularization/rkhs_norm": 216.00852966308594,
|
||
|
|
"rewards/chosen": 0.562893506448553,
|
||
|
|
"rewards/margins": 0.9334219640654239,
|
||
|
|
"rewards/rejected": -0.3705284576168709,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 11.130681991577148,
|
||
|
|
"kl": 12.841374397277832,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -34324413.424148604,
|
||
|
|
"logits/rejected": -35546018.32176656,
|
||
|
|
"logps/chosen": -491.4020897832817,
|
||
|
|
"logps/rejected": -352.90883773659306,
|
||
|
|
"loss": 0.5267,
|
||
|
|
"loss/base_kto": 3.0797278881073,
|
||
|
|
"metrics/advantage_var": 376.87994384765625,
|
||
|
|
"regularization/mmd": 1.1341991424560547,
|
||
|
|
"regularization/rkhs_norm": 218.5354766845703,
|
||
|
|
"rewards/chosen": 0.6077518049777477,
|
||
|
|
"rewards/margins": 0.95141431964929,
|
||
|
|
"rewards/rejected": -0.3436625146715423,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 11.328431129455566,
|
||
|
|
"kl": 11.874162673950195,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -35556658.89614243,
|
||
|
|
"logits/rejected": -35123673.13531353,
|
||
|
|
"logps/chosen": -468.75018545994067,
|
||
|
|
"logps/rejected": -391.64253300330034,
|
||
|
|
"loss": 0.5277,
|
||
|
|
"loss/base_kto": 3.1278867721557617,
|
||
|
|
"metrics/advantage_var": 333.9039001464844,
|
||
|
|
"regularization/mmd": 1.0934902429580688,
|
||
|
|
"regularization/rkhs_norm": 210.69175720214844,
|
||
|
|
"rewards/chosen": 0.5320587610986183,
|
||
|
|
"rewards/margins": 0.8782715699405377,
|
||
|
|
"rewards/rejected": -0.34621280884191935,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 8.911983489990234,
|
||
|
|
"kl": 15.25463581085205,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -35222911.174193546,
|
||
|
|
"logits/rejected": -36076807.75757576,
|
||
|
|
"logps/chosen": -480.13860887096774,
|
||
|
|
"logps/rejected": -394.96796875,
|
||
|
|
"loss": 0.5294,
|
||
|
|
"loss/base_kto": 3.02984619140625,
|
||
|
|
"metrics/advantage_var": 424.6706237792969,
|
||
|
|
"regularization/mmd": 1.2054380178451538,
|
||
|
|
"regularization/rkhs_norm": 232.2616729736328,
|
||
|
|
"rewards/chosen": 0.6062917893932712,
|
||
|
|
"rewards/margins": 1.0025537097442418,
|
||
|
|
"rewards/rejected": -0.39626192035097063,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 11.520490646362305,
|
||
|
|
"kl": 13.6519193649292,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -35128602.15479876,
|
||
|
|
"logits/rejected": -36351912.78233439,
|
||
|
|
"logps/chosen": -468.5253482972136,
|
||
|
|
"logps/rejected": -383.8808162460568,
|
||
|
|
"loss": 0.5307,
|
||
|
|
"loss/base_kto": 3.1344125270843506,
|
||
|
|
"metrics/advantage_var": 350.7089538574219,
|
||
|
|
"regularization/mmd": 1.1107876300811768,
|
||
|
|
"regularization/rkhs_norm": 214.02456665039062,
|
||
|
|
"rewards/chosen": 0.527235096084075,
|
||
|
|
"rewards/margins": 0.8736736754628951,
|
||
|
|
"rewards/rejected": -0.34643857937882,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 10.270307540893555,
|
||
|
|
"kl": 13.290621757507324,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -33216139.925465837,
|
||
|
|
"logits/rejected": -35478682.56603774,
|
||
|
|
"logps/chosen": -503.1929347826087,
|
||
|
|
"logps/rejected": -354.3146373820755,
|
||
|
|
"loss": 0.5281,
|
||
|
|
"loss/base_kto": 3.0898449420928955,
|
||
|
|
"metrics/advantage_var": 369.3459167480469,
|
||
|
|
"regularization/mmd": 1.1351157426834106,
|
||
|
|
"regularization/rkhs_norm": 218.7120819091797,
|
||
|
|
"rewards/chosen": 0.5763542222680512,
|
||
|
|
"rewards/margins": 0.9304242739291287,
|
||
|
|
"rewards/rejected": -0.3540700516610775,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 7.135807037353516,
|
||
|
|
"kl": 12.883994102478027,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -34329645.801916935,
|
||
|
|
"logits/rejected": -36721967.755351685,
|
||
|
|
"logps/chosen": -490.8725039936102,
|
||
|
|
"logps/rejected": -378.97720756880733,
|
||
|
|
"loss": 0.5265,
|
||
|
|
"loss/base_kto": 3.095879554748535,
|
||
|
|
"metrics/advantage_var": 361.8744812011719,
|
||
|
|
"regularization/mmd": 1.1162868738174438,
|
||
|
|
"regularization/rkhs_norm": 215.0841827392578,
|
||
|
|
"rewards/chosen": 0.5800625249600639,
|
||
|
|
"rewards/margins": 0.9294950473284221,
|
||
|
|
"rewards/rejected": -0.3494325223683582,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 10.335249900817871,
|
||
|
|
"kl": 12.950479507446289,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -34357906.977917984,
|
||
|
|
"logits/rejected": -35518349.86996904,
|
||
|
|
"logps/chosen": -489.25621056782336,
|
||
|
|
"logps/rejected": -372.02549342105266,
|
||
|
|
"loss": 0.5207,
|
||
|
|
"loss/base_kto": 3.0475614070892334,
|
||
|
|
"metrics/advantage_var": 361.4256286621094,
|
||
|
|
"regularization/mmd": 1.1180591583251953,
|
||
|
|
"regularization/rkhs_norm": 215.42568969726562,
|
||
|
|
"rewards/chosen": 0.5793135008225305,
|
||
|
|
"rewards/margins": 0.9720364189757087,
|
||
|
|
"rewards/rejected": -0.3927229181531782,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 12.101631164550781,
|
||
|
|
"kl": 14.51342487335205,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34178817.240710825,
|
||
|
|
"logits/rejected": -35369808.944024205,
|
||
|
|
"logps/chosen": -498.0520496768982,
|
||
|
|
"logps/rejected": -371.3050302571861,
|
||
|
|
"loss": 0.5303,
|
||
|
|
"loss/base_kto": 3.0979537963867188,
|
||
|
|
"metrics/advantage_var": 576.8146362304688,
|
||
|
|
"regularization/mmd": 1.1443445682525635,
|
||
|
|
"regularization/rkhs_norm": 220.4902801513672,
|
||
|
|
"rewards/chosen": 0.5742010014892972,
|
||
|
|
"rewards/margins": 0.8867016940213273,
|
||
|
|
"rewards/rejected": -0.31250069253203006,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 12.365646362304688,
|
||
|
|
"kl": 13.501253128051758,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -35068562.04590164,
|
||
|
|
"logits/rejected": -37147907.82089552,
|
||
|
|
"logps/chosen": -491.85056352459014,
|
||
|
|
"logps/rejected": -379.8675839552239,
|
||
|
|
"loss": 0.5213,
|
||
|
|
"loss/base_kto": 3.0109212398529053,
|
||
|
|
"metrics/advantage_var": 396.2356872558594,
|
||
|
|
"regularization/mmd": 1.1596250534057617,
|
||
|
|
"regularization/rkhs_norm": 223.43447875976562,
|
||
|
|
"rewards/chosen": 0.5979486184042008,
|
||
|
|
"rewards/margins": 0.9946309477243966,
|
||
|
|
"rewards/rejected": -0.39668232932019587,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 7.931318759918213,
|
||
|
|
"kl": 12.034379959106445,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -34815871.38164251,
|
||
|
|
"logits/rejected": -36582507.21699545,
|
||
|
|
"logps/chosen": -469.4171698872786,
|
||
|
|
"logps/rejected": -372.40629742033383,
|
||
|
|
"loss": 0.5308,
|
||
|
|
"loss/base_kto": 3.097630023956299,
|
||
|
|
"metrics/advantage_var": 385.7208557128906,
|
||
|
|
"regularization/mmd": 1.1488761901855469,
|
||
|
|
"regularization/rkhs_norm": 221.36343383789062,
|
||
|
|
"rewards/chosen": 0.5448376445186695,
|
||
|
|
"rewards/margins": 0.933819696663254,
|
||
|
|
"rewards/rejected": -0.3889820521445846,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 13.937902450561523,
|
||
|
|
"kl": 12.061327934265137,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -35316257.267828844,
|
||
|
|
"logits/rejected": -35816481.13405239,
|
||
|
|
"logps/chosen": -467.92115689381933,
|
||
|
|
"logps/rejected": -386.2687548151002,
|
||
|
|
"loss": 0.5311,
|
||
|
|
"loss/base_kto": 3.126477003097534,
|
||
|
|
"metrics/advantage_var": 370.1043395996094,
|
||
|
|
"regularization/mmd": 1.1222646236419678,
|
||
|
|
"regularization/rkhs_norm": 216.23594665527344,
|
||
|
|
"rewards/chosen": 0.518689138953546,
|
||
|
|
"rewards/margins": 0.8781875258949943,
|
||
|
|
"rewards/rejected": -0.35949838694144837,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 9.281513214111328,
|
||
|
|
"kl": 12.302824974060059,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -35342033.45454545,
|
||
|
|
"logits/rejected": -36803324.915662654,
|
||
|
|
"logps/chosen": -493.375101461039,
|
||
|
|
"logps/rejected": -401.9391942771084,
|
||
|
|
"loss": 0.5236,
|
||
|
|
"loss/base_kto": 3.0720460414886475,
|
||
|
|
"metrics/advantage_var": 353.0370178222656,
|
||
|
|
"regularization/mmd": 1.1166185140609741,
|
||
|
|
"regularization/rkhs_norm": 215.14805603027344,
|
||
|
|
"rewards/chosen": 0.5612790987089082,
|
||
|
|
"rewards/margins": 0.9515410130655535,
|
||
|
|
"rewards/rejected": -0.3902619143566453,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 11.074307441711426,
|
||
|
|
"kl": 10.97407054901123,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -34191000.47648903,
|
||
|
|
"logits/rejected": -35948942.75389408,
|
||
|
|
"logps/chosen": -464.1570336990596,
|
||
|
|
"logps/rejected": -365.73203855140184,
|
||
|
|
"loss": 0.5278,
|
||
|
|
"loss/base_kto": 3.1273510456085205,
|
||
|
|
"metrics/advantage_var": 343.7378845214844,
|
||
|
|
"regularization/mmd": 1.0950945615768433,
|
||
|
|
"regularization/rkhs_norm": 211.0008544921875,
|
||
|
|
"rewards/chosen": 0.5045455406452047,
|
||
|
|
"rewards/margins": 0.8785536242858054,
|
||
|
|
"rewards/rejected": -0.3740080836406007,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 10.354769706726074,
|
||
|
|
"kl": 11.708769798278809,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -35112156.67596899,
|
||
|
|
"logits/rejected": -36165696.503937006,
|
||
|
|
"logps/chosen": -493.58856589147285,
|
||
|
|
"logps/rejected": -394.29537401574805,
|
||
|
|
"loss": 0.5327,
|
||
|
|
"loss/base_kto": 3.140596389770508,
|
||
|
|
"metrics/advantage_var": 366.3534240722656,
|
||
|
|
"regularization/mmd": 1.1208542585372925,
|
||
|
|
"regularization/rkhs_norm": 215.96421813964844,
|
||
|
|
"rewards/chosen": 0.5278361948885659,
|
||
|
|
"rewards/margins": 0.8934785923110561,
|
||
|
|
"rewards/rejected": -0.36564239742249016,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 7.407812595367432,
|
||
|
|
"kl": 9.797353744506836,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -34317422.32473445,
|
||
|
|
"logits/rejected": -33990174.50563607,
|
||
|
|
"logps/chosen": -469.95428679817906,
|
||
|
|
"logps/rejected": -343.463768115942,
|
||
|
|
"loss": 0.5238,
|
||
|
|
"loss/base_kto": 3.1010022163391113,
|
||
|
|
"metrics/advantage_var": 348.421142578125,
|
||
|
|
"regularization/mmd": 1.0893239974975586,
|
||
|
|
"regularization/rkhs_norm": 209.88902282714844,
|
||
|
|
"rewards/chosen": 0.5466918019134105,
|
||
|
|
"rewards/margins": 0.9251473647813906,
|
||
|
|
"rewards/rejected": -0.3784555628679801,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.975174191323546e+17,
|
||
|
|
"train_loss": 0.5582944930938035,
|
||
|
|
"train_runtime": 11062.9397,
|
||
|
|
"train_samples_per_second": 13.398,
|
||
|
|
"train_steps_per_second": 0.209
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.975174191323546e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|