1513055e69
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed4 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 31.407852172851562,
|
|
"kl": 21.7319393157959,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35708672.0,
|
|
"logits/rejected": -37165390.76923077,
|
|
"logps/chosen": -496.9298780487805,
|
|
"logps/rejected": -362.92918669871796,
|
|
"loss": 0.5378,
|
|
"loss/base_kto": 3.8584771156311035,
|
|
"loss/total_with_kl": 4.3021464347839355,
|
|
"metrics/advantage_var": 266.4680480957031,
|
|
"regularization/advantage_var": 266.4680480957031,
|
|
"regularization/kl": 0.44366922974586487,
|
|
"rewards/chosen": 0.3604372303660323,
|
|
"rewards/margins": 0.13938797049555204,
|
|
"rewards/rejected": 0.22104925987048027,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 13.991652488708496,
|
|
"kl": 11.298606872558594,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36759266.597243495,
|
|
"logits/rejected": -36918110.315789476,
|
|
"logps/chosen": -483.4872702909648,
|
|
"logps/rejected": -404.06678628389153,
|
|
"loss": 0.5262,
|
|
"loss/base_kto": 3.8981258869171143,
|
|
"loss/total_with_kl": 4.209258079528809,
|
|
"metrics/advantage_var": 186.8661346435547,
|
|
"regularization/advantage_var": 186.8661346435547,
|
|
"regularization/kl": 0.3111321032047272,
|
|
"rewards/chosen": 0.1703898968776919,
|
|
"rewards/margins": 0.10665024803134397,
|
|
"rewards/rejected": 0.06373964884634793,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 14.879023551940918,
|
|
"kl": 16.563892364501953,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35328210.131699845,
|
|
"logits/rejected": -36644468.77192982,
|
|
"logps/chosen": -492.49081163859114,
|
|
"logps/rejected": -387.8072169059011,
|
|
"loss": 0.5241,
|
|
"loss/base_kto": 3.8087785243988037,
|
|
"loss/total_with_kl": 4.192681789398193,
|
|
"metrics/advantage_var": 230.5727081298828,
|
|
"regularization/advantage_var": 230.5727081298828,
|
|
"regularization/kl": 0.3839035630226135,
|
|
"rewards/chosen": 0.33150058062846954,
|
|
"rewards/margins": 0.168357088046918,
|
|
"rewards/rejected": 0.16314349258155153,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 10.058321952819824,
|
|
"kl": 4.800376892089844,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37254383.51186441,
|
|
"logits/rejected": -37743794.08695652,
|
|
"logps/chosen": -489.9477224576271,
|
|
"logps/rejected": -394.09415760869564,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.8233535289764404,
|
|
"loss/total_with_kl": 4.199948787689209,
|
|
"metrics/advantage_var": 226.18325805664062,
|
|
"regularization/advantage_var": 226.18325805664062,
|
|
"regularization/kl": 0.3765951097011566,
|
|
"rewards/chosen": -0.06402210946810448,
|
|
"rewards/margins": 0.15035100640315913,
|
|
"rewards/rejected": -0.2143731158712636,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 14.036251068115234,
|
|
"kl": 6.662667751312256,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35068959.80124223,
|
|
"logits/rejected": -36745950.18867925,
|
|
"logps/chosen": -482.4549204192547,
|
|
"logps/rejected": -355.26459316037733,
|
|
"loss": 0.5194,
|
|
"loss/base_kto": 3.804851531982422,
|
|
"loss/total_with_kl": 4.155176639556885,
|
|
"metrics/advantage_var": 210.4054718017578,
|
|
"regularization/advantage_var": 210.4054718017578,
|
|
"regularization/kl": 0.3503250777721405,
|
|
"rewards/chosen": 0.019241907581779526,
|
|
"rewards/margins": 0.19215162622122528,
|
|
"rewards/rejected": -0.17290971863944576,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 17.4003849029541,
|
|
"kl": 17.45867919921875,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36515702.594249204,
|
|
"logits/rejected": -36474366.434250765,
|
|
"logps/chosen": -468.5167731629393,
|
|
"logps/rejected": -374.1554376911315,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.8414978981018066,
|
|
"loss/total_with_kl": 4.200013637542725,
|
|
"metrics/advantage_var": 215.3249053955078,
|
|
"regularization/advantage_var": 215.3249053955078,
|
|
"regularization/kl": 0.3585159480571747,
|
|
"rewards/chosen": 0.3180838880447534,
|
|
"rewards/margins": 0.13438673291765096,
|
|
"rewards/rejected": 0.18369715512710244,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 15.743810653686523,
|
|
"kl": 10.514762878417969,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36569637.38897169,
|
|
"logits/rejected": -37995814.25287356,
|
|
"logps/chosen": -496.6527570789866,
|
|
"logps/rejected": -369.03014675697864,
|
|
"loss": 0.5272,
|
|
"loss/base_kto": 3.764463186264038,
|
|
"loss/total_with_kl": 4.2173614501953125,
|
|
"metrics/advantage_var": 272.01068115234375,
|
|
"regularization/advantage_var": 272.01068115234375,
|
|
"regularization/kl": 0.4528977572917938,
|
|
"rewards/chosen": 0.17616865496344075,
|
|
"rewards/margins": 0.2380280113221615,
|
|
"rewards/rejected": -0.06185935635872075,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 13.376973152160645,
|
|
"kl": 11.327672004699707,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36204002.461538464,
|
|
"logits/rejected": -36326381.26829268,
|
|
"logps/chosen": -476.1418269230769,
|
|
"logps/rejected": -349.2894912347561,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.762291669845581,
|
|
"loss/total_with_kl": 4.127110958099365,
|
|
"metrics/advantage_var": 219.1105194091797,
|
|
"regularization/advantage_var": 219.1105194091797,
|
|
"regularization/kl": 0.3648190200328827,
|
|
"rewards/chosen": 0.1949013685568785,
|
|
"rewards/margins": 0.23164148260609219,
|
|
"rewards/rejected": -0.036740114049213686,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 12.790648460388184,
|
|
"kl": 3.0852344036102295,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35509029.822706066,
|
|
"logits/rejected": -36370239.09576138,
|
|
"logps/chosen": -454.6269440124417,
|
|
"logps/rejected": -361.42177688383043,
|
|
"loss": 0.5176,
|
|
"loss/base_kto": 3.7811639308929443,
|
|
"loss/total_with_kl": 4.141047954559326,
|
|
"metrics/advantage_var": 216.1467742919922,
|
|
"regularization/advantage_var": 216.1467742919922,
|
|
"regularization/kl": 0.3598843812942505,
|
|
"rewards/chosen": 0.03764512935656226,
|
|
"rewards/margins": 0.22475955149448293,
|
|
"rewards/rejected": -0.18711442213792068,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 13.26974868774414,
|
|
"kl": 9.794063568115234,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -35194075.08082409,
|
|
"logits/rejected": -38104267.537750386,
|
|
"logps/chosen": -497.2285063391442,
|
|
"logps/rejected": -373.55662557781204,
|
|
"loss": 0.5216,
|
|
"loss/base_kto": 3.756938934326172,
|
|
"loss/total_with_kl": 4.173141002655029,
|
|
"metrics/advantage_var": 249.9713134765625,
|
|
"regularization/advantage_var": 249.9713134765625,
|
|
"regularization/kl": 0.41620221734046936,
|
|
"rewards/chosen": 0.1574949817687698,
|
|
"rewards/margins": 0.2115917268174926,
|
|
"rewards/rejected": -0.05409674504872279,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 14.107484817504883,
|
|
"kl": 6.901716709136963,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36784849.491311215,
|
|
"logits/rejected": -36821523.78361669,
|
|
"logps/chosen": -480.4296998420221,
|
|
"logps/rejected": -375.8837905718702,
|
|
"loss": 0.5297,
|
|
"loss/base_kto": 3.738966464996338,
|
|
"loss/total_with_kl": 4.237467288970947,
|
|
"metrics/advantage_var": 299.39990234375,
|
|
"regularization/advantage_var": 299.39990234375,
|
|
"regularization/kl": 0.4985008239746094,
|
|
"rewards/chosen": 0.1235669032092343,
|
|
"rewards/margins": 0.2720938333918077,
|
|
"rewards/rejected": -0.1485269301825734,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 16.072452545166016,
|
|
"kl": 14.483355522155762,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36794460.080620155,
|
|
"logits/rejected": -36787279.01732284,
|
|
"logps/chosen": -472.3289728682171,
|
|
"logps/rejected": -384.60944881889765,
|
|
"loss": 0.5219,
|
|
"loss/base_kto": 3.738342046737671,
|
|
"loss/total_with_kl": 4.175151348114014,
|
|
"metrics/advantage_var": 262.34765625,
|
|
"regularization/advantage_var": 262.34765625,
|
|
"regularization/kl": 0.43680888414382935,
|
|
"rewards/chosen": 0.26270316663638565,
|
|
"rewards/margins": 0.2332827044560753,
|
|
"rewards/rejected": 0.029420462180310348,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 31.844099044799805,
|
|
"kl": 18.43171501159668,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36242617.35311573,
|
|
"logits/rejected": -38018287.94719472,
|
|
"logps/chosen": -478.20938427299706,
|
|
"logps/rejected": -370.2881342821782,
|
|
"loss": 0.5163,
|
|
"loss/base_kto": 3.638079881668091,
|
|
"loss/total_with_kl": 4.130200386047363,
|
|
"metrics/advantage_var": 295.5679626464844,
|
|
"regularization/advantage_var": 295.5679626464844,
|
|
"regularization/kl": 0.4921206533908844,
|
|
"rewards/chosen": 0.44125633352941857,
|
|
"rewards/margins": 0.3134749534016597,
|
|
"rewards/rejected": 0.12778138012775886,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 23.6060848236084,
|
|
"kl": 20.614234924316406,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -38277793.39969834,
|
|
"logits/rejected": -39959278.158833064,
|
|
"logps/chosen": -489.9046003016591,
|
|
"logps/rejected": -354.2591166936791,
|
|
"loss": 0.5232,
|
|
"loss/base_kto": 3.713111639022827,
|
|
"loss/total_with_kl": 4.185714244842529,
|
|
"metrics/advantage_var": 283.8451843261719,
|
|
"regularization/advantage_var": 283.8451843261719,
|
|
"regularization/kl": 0.4726021885871887,
|
|
"rewards/chosen": 0.3984208833398143,
|
|
"rewards/margins": 0.2593852828290579,
|
|
"rewards/rejected": 0.13903560051075642,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 13.035717964172363,
|
|
"kl": 25.790876388549805,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37029192.39178515,
|
|
"logits/rejected": -38328598.55332303,
|
|
"logps/chosen": -489.5135268562401,
|
|
"logps/rejected": -380.8914702472952,
|
|
"loss": 0.5228,
|
|
"loss/base_kto": 3.7213408946990967,
|
|
"loss/total_with_kl": 4.182716369628906,
|
|
"metrics/advantage_var": 277.1021423339844,
|
|
"regularization/advantage_var": 277.1021423339844,
|
|
"regularization/kl": 0.46137505769729614,
|
|
"rewards/chosen": 0.5010644511965591,
|
|
"rewards/margins": 0.25748866081486954,
|
|
"rewards/rejected": 0.24357579038168953,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 15.439168930053711,
|
|
"kl": 19.15919303894043,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -38340044.6443769,
|
|
"logits/rejected": -39156298.08360129,
|
|
"logps/chosen": -493.11816109422494,
|
|
"logps/rejected": -358.1865454180064,
|
|
"loss": 0.5118,
|
|
"loss/base_kto": 3.6858232021331787,
|
|
"loss/total_with_kl": 4.094377040863037,
|
|
"metrics/advantage_var": 245.37767028808594,
|
|
"regularization/advantage_var": 245.37767028808594,
|
|
"regularization/kl": 0.408553808927536,
|
|
"rewards/chosen": 0.3816912340900456,
|
|
"rewards/margins": 0.2520246695411483,
|
|
"rewards/rejected": 0.12966656454889722,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 15.302995681762695,
|
|
"kl": 19.191791534423828,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36681801.9466248,
|
|
"logits/rejected": -38200022.19595645,
|
|
"logps/chosen": -474.67268445839875,
|
|
"logps/rejected": -362.0720985614308,
|
|
"loss": 0.5166,
|
|
"loss/base_kto": 3.708709478378296,
|
|
"loss/total_with_kl": 4.133121490478516,
|
|
"metrics/advantage_var": 254.9022674560547,
|
|
"regularization/advantage_var": 254.9022674560547,
|
|
"regularization/kl": 0.4244123101234436,
|
|
"rewards/chosen": 0.2618728949099171,
|
|
"rewards/margins": 0.26020920784020074,
|
|
"rewards/rejected": 0.0016636870697163868,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 14.084619522094727,
|
|
"kl": 21.147933959960938,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -37405050.67488444,
|
|
"logits/rejected": -38261949.87004755,
|
|
"logps/chosen": -484.3794780431433,
|
|
"logps/rejected": -385.4020156497623,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.6717727184295654,
|
|
"loss/total_with_kl": 4.127252101898193,
|
|
"metrics/advantage_var": 273.56134033203125,
|
|
"regularization/advantage_var": 273.56134033203125,
|
|
"regularization/kl": 0.45547962188720703,
|
|
"rewards/chosen": 0.3963017397558744,
|
|
"rewards/margins": 0.26100734088735594,
|
|
"rewards/rejected": 0.13529439886851846,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 12.959440231323242,
|
|
"kl": 19.34718894958496,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35951478.12405446,
|
|
"logits/rejected": -39251895.21163166,
|
|
"logps/chosen": -501.0877458396369,
|
|
"logps/rejected": -371.3808057350565,
|
|
"loss": 0.5161,
|
|
"loss/base_kto": 3.6851253509521484,
|
|
"loss/total_with_kl": 4.128602504730225,
|
|
"metrics/advantage_var": 266.3529052734375,
|
|
"regularization/advantage_var": 266.3529052734375,
|
|
"regularization/kl": 0.4434775412082672,
|
|
"rewards/chosen": 0.3878343729316377,
|
|
"rewards/margins": 0.267031476650779,
|
|
"rewards/rejected": 0.12080289628085875,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 14.978385925292969,
|
|
"kl": 18.645681381225586,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36797240.93312597,
|
|
"logits/rejected": -38115401.9466248,
|
|
"logps/chosen": -460.13826788491446,
|
|
"logps/rejected": -361.6690296310832,
|
|
"loss": 0.5169,
|
|
"loss/base_kto": 3.7090961933135986,
|
|
"loss/total_with_kl": 4.134805202484131,
|
|
"metrics/advantage_var": 255.6807861328125,
|
|
"regularization/advantage_var": 255.6807861328125,
|
|
"regularization/kl": 0.425708532333374,
|
|
"rewards/chosen": 0.3230946067703271,
|
|
"rewards/margins": 0.2698136521249097,
|
|
"rewards/rejected": 0.053280954645417364,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 19.60897445678711,
|
|
"kl": 9.722540855407715,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35991659.61247948,
|
|
"logits/rejected": -37533399.94038748,
|
|
"logps/chosen": -472.529197454844,
|
|
"logps/rejected": -352.2409649776453,
|
|
"loss": 0.5154,
|
|
"loss/base_kto": 3.6974093914031982,
|
|
"loss/total_with_kl": 4.123062610626221,
|
|
"metrics/advantage_var": 255.6476593017578,
|
|
"regularization/advantage_var": 255.6476593017578,
|
|
"regularization/kl": 0.4256533682346344,
|
|
"rewards/chosen": 0.21491746323057778,
|
|
"rewards/margins": 0.28975284113552374,
|
|
"rewards/rejected": -0.07483537790494597,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 16.646005630493164,
|
|
"kl": 8.006230354309082,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37325941.915151514,
|
|
"logits/rejected": -37923820.18064516,
|
|
"logps/chosen": -508.9840909090909,
|
|
"logps/rejected": -372.57459677419354,
|
|
"loss": 0.5222,
|
|
"loss/base_kto": 3.70198130607605,
|
|
"loss/total_with_kl": 4.177700042724609,
|
|
"metrics/advantage_var": 285.7170104980469,
|
|
"regularization/advantage_var": 285.7170104980469,
|
|
"regularization/kl": 0.47571879625320435,
|
|
"rewards/chosen": 0.11891341931892163,
|
|
"rewards/margins": 0.2911074264541283,
|
|
"rewards/rejected": -0.17219400713520666,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 14.671125411987305,
|
|
"kl": 9.332090377807617,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -37215868.35510888,
|
|
"logits/rejected": -37923514.65885798,
|
|
"logps/chosen": -471.71869765494137,
|
|
"logps/rejected": -402.11786237188875,
|
|
"loss": 0.5121,
|
|
"loss/base_kto": 3.6447296142578125,
|
|
"loss/total_with_kl": 4.097075939178467,
|
|
"metrics/advantage_var": 271.67950439453125,
|
|
"regularization/advantage_var": 271.67950439453125,
|
|
"regularization/kl": 0.4523463249206543,
|
|
"rewards/chosen": 0.09727847995470516,
|
|
"rewards/margins": 0.316666735217848,
|
|
"rewards/rejected": -0.21938825526314285,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 20.42093849182129,
|
|
"kl": 8.731324195861816,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36552382.12634823,
|
|
"logits/rejected": -37327782.74484944,
|
|
"logps/chosen": -496.2094568567026,
|
|
"logps/rejected": -372.47043383518223,
|
|
"loss": 0.5237,
|
|
"loss/base_kto": 3.6933434009552,
|
|
"loss/total_with_kl": 4.189201354980469,
|
|
"metrics/advantage_var": 297.812744140625,
|
|
"regularization/advantage_var": 297.812744140625,
|
|
"regularization/kl": 0.49585819244384766,
|
|
"rewards/chosen": 0.14480564957958156,
|
|
"rewards/margins": 0.28422389942017434,
|
|
"rewards/rejected": -0.1394182498405928,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 15.883015632629395,
|
|
"kl": 6.376058101654053,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -34663966.51655629,
|
|
"logits/rejected": -36241871.52662722,
|
|
"logps/chosen": -490.091059602649,
|
|
"logps/rejected": -375.5445173816568,
|
|
"loss": 0.5119,
|
|
"loss/base_kto": 3.6528472900390625,
|
|
"loss/total_with_kl": 4.094952583312988,
|
|
"metrics/advantage_var": 265.52880859375,
|
|
"regularization/advantage_var": 265.52880859375,
|
|
"regularization/kl": 0.4421055018901825,
|
|
"rewards/chosen": 0.04661942791465103,
|
|
"rewards/margins": 0.31981497679044846,
|
|
"rewards/rejected": -0.27319554887579744,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 12.917749404907227,
|
|
"kl": 8.632678031921387,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36128419.469879515,
|
|
"logits/rejected": -36053909.61038961,
|
|
"logps/chosen": -509.4839043674699,
|
|
"logps/rejected": -378.4122615665584,
|
|
"loss": 0.5256,
|
|
"loss/base_kto": 3.6626203060150146,
|
|
"loss/total_with_kl": 4.205124855041504,
|
|
"metrics/advantage_var": 325.8285827636719,
|
|
"regularization/advantage_var": 325.8285827636719,
|
|
"regularization/kl": 0.5425044894218445,
|
|
"rewards/chosen": 0.13677340817738728,
|
|
"rewards/margins": 0.3419107352972889,
|
|
"rewards/rejected": -0.20513732711990157,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 10.976865768432617,
|
|
"kl": 10.955889701843262,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36438100.65500795,
|
|
"logits/rejected": -36357725.59139785,
|
|
"logps/chosen": -496.5558426073132,
|
|
"logps/rejected": -369.4063460061444,
|
|
"loss": 0.5163,
|
|
"loss/base_kto": 3.6595475673675537,
|
|
"loss/total_with_kl": 4.130549430847168,
|
|
"metrics/advantage_var": 282.8838806152344,
|
|
"regularization/advantage_var": 282.8838806152344,
|
|
"regularization/kl": 0.4710017144680023,
|
|
"rewards/chosen": 0.18982798420188915,
|
|
"rewards/margins": 0.29242409722812285,
|
|
"rewards/rejected": -0.10259611302623368,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 14.063850402832031,
|
|
"kl": 8.425477981567383,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35098068.720893145,
|
|
"logits/rejected": -35861402.07044411,
|
|
"logps/chosen": -480.1312799043062,
|
|
"logps/rejected": -362.8375765696784,
|
|
"loss": 0.5184,
|
|
"loss/base_kto": 3.654953718185425,
|
|
"loss/total_with_kl": 4.147418022155762,
|
|
"metrics/advantage_var": 295.7740173339844,
|
|
"regularization/advantage_var": 295.7740173339844,
|
|
"regularization/kl": 0.49246373772621155,
|
|
"rewards/chosen": 0.08376480175547622,
|
|
"rewards/margins": 0.3313756445902293,
|
|
"rewards/rejected": -0.24761084283475307,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 22.535619735717773,
|
|
"kl": 5.692698955535889,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -33987992.63849765,
|
|
"logits/rejected": -36481305.160686426,
|
|
"logps/chosen": -473.60450899843505,
|
|
"logps/rejected": -393.6680479719189,
|
|
"loss": 0.5261,
|
|
"loss/base_kto": 3.7334465980529785,
|
|
"loss/total_with_kl": 4.208803176879883,
|
|
"metrics/advantage_var": 285.4994201660156,
|
|
"regularization/advantage_var": 285.4994201660156,
|
|
"regularization/kl": 0.4753565490245819,
|
|
"rewards/chosen": -0.007258453279593741,
|
|
"rewards/margins": 0.25475685521360536,
|
|
"rewards/rejected": -0.2620153084931991,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 14.898407936096191,
|
|
"kl": 7.375546455383301,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37071323.136,
|
|
"logits/rejected": -37126512.95267176,
|
|
"logps/chosen": -468.7284,
|
|
"logps/rejected": -398.7224952290076,
|
|
"loss": 0.5138,
|
|
"loss/base_kto": 3.680459976196289,
|
|
"loss/total_with_kl": 4.110528469085693,
|
|
"metrics/advantage_var": 258.29962158203125,
|
|
"regularization/advantage_var": 258.29962158203125,
|
|
"regularization/kl": 0.43006888031959534,
|
|
"rewards/chosen": 0.068622119140625,
|
|
"rewards/margins": 0.30442510883826335,
|
|
"rewards/rejected": -0.23580298969763835,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 14.685869216918945,
|
|
"kl": 7.301432132720947,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -34996881.21880065,
|
|
"logits/rejected": -36247459.330316745,
|
|
"logps/chosen": -494.2022893030794,
|
|
"logps/rejected": -366.40346907993967,
|
|
"loss": 0.5117,
|
|
"loss/base_kto": 3.652690887451172,
|
|
"loss/total_with_kl": 4.093989849090576,
|
|
"metrics/advantage_var": 265.0446472167969,
|
|
"regularization/advantage_var": 265.0446472167969,
|
|
"regularization/kl": 0.4412992596626282,
|
|
"rewards/chosen": 0.06706471497274488,
|
|
"rewards/margins": 0.3209760282210009,
|
|
"rewards/rejected": -0.25391131324825605,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 17.012447357177734,
|
|
"kl": 6.321325778961182,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -34797348.79510704,
|
|
"logits/rejected": -36428875.24600639,
|
|
"logps/chosen": -486.8264525993884,
|
|
"logps/rejected": -369.2567392172524,
|
|
"loss": 0.514,
|
|
"loss/base_kto": 3.7007968425750732,
|
|
"loss/total_with_kl": 4.11183500289917,
|
|
"metrics/advantage_var": 246.86985778808594,
|
|
"regularization/advantage_var": 246.86985778808594,
|
|
"regularization/kl": 0.4110383093357086,
|
|
"rewards/chosen": 0.08572891655318234,
|
|
"rewards/margins": 0.29614313271357573,
|
|
"rewards/rejected": -0.21041421616039338,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 21.558069229125977,
|
|
"kl": 6.568594455718994,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36229036.27672956,
|
|
"logits/rejected": -34802853.36645963,
|
|
"logps/chosen": -477.03586871069183,
|
|
"logps/rejected": -385.36818225931677,
|
|
"loss": 0.5199,
|
|
"loss/base_kto": 3.7086777687072754,
|
|
"loss/total_with_kl": 4.158957004547119,
|
|
"metrics/advantage_var": 270.4382019042969,
|
|
"regularization/advantage_var": 270.4382019042969,
|
|
"regularization/kl": 0.45027956366539,
|
|
"rewards/chosen": 0.024344616716013016,
|
|
"rewards/margins": 0.27192017520664774,
|
|
"rewards/rejected": -0.2475755584906347,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 15.986832618713379,
|
|
"kl": 10.523635864257812,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35453926.559006214,
|
|
"logits/rejected": -36066287.89937107,
|
|
"logps/chosen": -488.52135093167703,
|
|
"logps/rejected": -365.40109080188677,
|
|
"loss": 0.5114,
|
|
"loss/base_kto": 3.6973044872283936,
|
|
"loss/total_with_kl": 4.0909857749938965,
|
|
"metrics/advantage_var": 236.445068359375,
|
|
"regularization/advantage_var": 236.445068359375,
|
|
"regularization/kl": 0.39368098974227905,
|
|
"rewards/chosen": 0.19559511931046195,
|
|
"rewards/margins": 0.2771852259626798,
|
|
"rewards/rejected": -0.08159010665221784,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 16.754920959472656,
|
|
"kl": 12.260619163513184,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36413636.8620155,
|
|
"logits/rejected": -36617994.88503937,
|
|
"logps/chosen": -489.39670542635656,
|
|
"logps/rejected": -366.496062992126,
|
|
"loss": 0.5106,
|
|
"loss/base_kto": 3.6487183570861816,
|
|
"loss/total_with_kl": 4.084438800811768,
|
|
"metrics/advantage_var": 261.6939697265625,
|
|
"regularization/advantage_var": 261.6939697265625,
|
|
"regularization/kl": 0.43572044372558594,
|
|
"rewards/chosen": 0.2697226354318072,
|
|
"rewards/margins": 0.3251430919651641,
|
|
"rewards/rejected": -0.055420456533356914,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 13.55805778503418,
|
|
"kl": 15.827859878540039,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36800989.97730956,
|
|
"logits/rejected": -37554994.582202114,
|
|
"logps/chosen": -479.0614870340357,
|
|
"logps/rejected": -384.43071266968326,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.6306519508361816,
|
|
"loss/total_with_kl": 4.144779205322266,
|
|
"metrics/advantage_var": 308.7852478027344,
|
|
"regularization/advantage_var": 308.7852478027344,
|
|
"regularization/kl": 0.5141274333000183,
|
|
"rewards/chosen": 0.2789377122875937,
|
|
"rewards/margins": 0.3369197413465881,
|
|
"rewards/rejected": -0.05798202905899439,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 13.294617652893066,
|
|
"kl": 7.158412933349609,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36268843.47169811,
|
|
"logits/rejected": -36132946.68322981,
|
|
"logps/chosen": -471.4134728773585,
|
|
"logps/rejected": -367.129173136646,
|
|
"loss": 0.5121,
|
|
"loss/base_kto": 3.6745097637176514,
|
|
"loss/total_with_kl": 4.096756935119629,
|
|
"metrics/advantage_var": 253.6019287109375,
|
|
"regularization/advantage_var": 253.6019287109375,
|
|
"regularization/kl": 0.4222472310066223,
|
|
"rewards/chosen": 0.002667512533799657,
|
|
"rewards/margins": 0.2960335360061769,
|
|
"rewards/rejected": -0.2933660234723772,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 13.791971206665039,
|
|
"kl": 12.70384407043457,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35769428.69172932,
|
|
"logits/rejected": -37067341.42439024,
|
|
"logps/chosen": -503.9091165413534,
|
|
"logps/rejected": -361.11981707317074,
|
|
"loss": 0.508,
|
|
"loss/base_kto": 3.6423072814941406,
|
|
"loss/total_with_kl": 4.063610553741455,
|
|
"metrics/advantage_var": 253.0349884033203,
|
|
"regularization/advantage_var": 253.0349884033203,
|
|
"regularization/kl": 0.42130327224731445,
|
|
"rewards/chosen": 0.24071848016036185,
|
|
"rewards/margins": 0.32390460959856054,
|
|
"rewards/rejected": -0.08318612943819868,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 23.122209548950195,
|
|
"kl": 18.913740158081055,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36850221.969040245,
|
|
"logits/rejected": -37901305.51898734,
|
|
"logps/chosen": -468.94465944272446,
|
|
"logps/rejected": -361.54361155063293,
|
|
"loss": 0.5082,
|
|
"loss/base_kto": 3.410432815551758,
|
|
"loss/total_with_kl": 4.065714359283447,
|
|
"metrics/advantage_var": 393.5623779296875,
|
|
"regularization/advantage_var": 393.5623779296875,
|
|
"regularization/kl": 0.6552813649177551,
|
|
"rewards/chosen": 0.5387757009027913,
|
|
"rewards/margins": 0.5983267829827149,
|
|
"rewards/rejected": -0.05955108207992361,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 17.803544998168945,
|
|
"kl": 12.378803253173828,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35954840.07920792,
|
|
"logits/rejected": -37374268.01186944,
|
|
"logps/chosen": -469.9499793729373,
|
|
"logps/rejected": -365.42447143916917,
|
|
"loss": 0.5176,
|
|
"loss/base_kto": 3.1306824684143066,
|
|
"loss/total_with_kl": 4.140599250793457,
|
|
"metrics/advantage_var": 606.5565795898438,
|
|
"regularization/advantage_var": 606.5565795898438,
|
|
"regularization/kl": 1.0099166631698608,
|
|
"rewards/chosen": 0.5161125762234426,
|
|
"rewards/margins": 0.9470148026120739,
|
|
"rewards/rejected": -0.4309022263886313,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 30.049779891967773,
|
|
"kl": 10.264975547790527,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -33420564.625592418,
|
|
"logits/rejected": -35505707.52395672,
|
|
"logps/chosen": -463.80835308056874,
|
|
"logps/rejected": -381.7633066074189,
|
|
"loss": 0.5134,
|
|
"loss/base_kto": 3.1224896907806396,
|
|
"loss/total_with_kl": 4.1072797775268555,
|
|
"metrics/advantage_var": 591.4656372070312,
|
|
"regularization/advantage_var": 591.4656372070312,
|
|
"regularization/kl": 0.9847902655601501,
|
|
"rewards/chosen": 0.5197738478722354,
|
|
"rewards/margins": 0.9656515126652705,
|
|
"rewards/rejected": -0.44587766479303514,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 21.001802444458008,
|
|
"kl": 9.1188325881958,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35266192.41025641,
|
|
"logits/rejected": -37204579.90243903,
|
|
"logps/chosen": -486.75320512820514,
|
|
"logps/rejected": -379.7540015243902,
|
|
"loss": 0.5272,
|
|
"loss/base_kto": 3.172616958618164,
|
|
"loss/total_with_kl": 4.2177653312683105,
|
|
"metrics/advantage_var": 627.7164916992188,
|
|
"regularization/advantage_var": 627.7164916992188,
|
|
"regularization/kl": 1.0451478958129883,
|
|
"rewards/chosen": 0.4424891838660607,
|
|
"rewards/margins": 0.9121579211379975,
|
|
"rewards/rejected": -0.4696687372719369,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 19.123069763183594,
|
|
"kl": 8.992833137512207,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34572778.09191759,
|
|
"logits/rejected": -35946439.198767334,
|
|
"logps/chosen": -473.49583993660855,
|
|
"logps/rejected": -376.42221205701077,
|
|
"loss": 0.5479,
|
|
"loss/base_kto": 3.2300808429718018,
|
|
"loss/total_with_kl": 4.383556365966797,
|
|
"metrics/advantage_var": 692.7782592773438,
|
|
"regularization/advantage_var": 692.7782592773438,
|
|
"regularization/kl": 1.1534757614135742,
|
|
"rewards/chosen": 0.33582137843886195,
|
|
"rewards/margins": 0.8600832762503989,
|
|
"rewards/rejected": -0.524261897811537,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 25.682153701782227,
|
|
"kl": 12.340991020202637,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -33759355.1392405,
|
|
"logits/rejected": -35178097.777777776,
|
|
"logps/chosen": -456.51864121835445,
|
|
"logps/rejected": -369.23042052469134,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.1937670707702637,
|
|
"loss/total_with_kl": 4.236794948577881,
|
|
"metrics/advantage_var": 626.4430541992188,
|
|
"regularization/advantage_var": 626.4430541992188,
|
|
"regularization/kl": 1.0430277585983276,
|
|
"rewards/chosen": 0.5073145130012608,
|
|
"rewards/margins": 0.8919978865349101,
|
|
"rewards/rejected": -0.38468337353364923,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 14.437877655029297,
|
|
"kl": 12.907342910766602,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34224996.49283668,
|
|
"logits/rejected": -33979712.219931275,
|
|
"logps/chosen": -463.4241583094556,
|
|
"logps/rejected": -374.91771370274915,
|
|
"loss": 0.5354,
|
|
"loss/base_kto": 3.1738390922546387,
|
|
"loss/total_with_kl": 4.2835001945495605,
|
|
"metrics/advantage_var": 666.4631958007812,
|
|
"regularization/advantage_var": 666.4631958007812,
|
|
"regularization/kl": 1.1096612215042114,
|
|
"rewards/chosen": 0.5474070901515491,
|
|
"rewards/margins": 0.8528692227660519,
|
|
"rewards/rejected": -0.3054621326145028,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 17.568225860595703,
|
|
"kl": 18.759857177734375,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34270990.29027113,
|
|
"logits/rejected": -36013280.24502297,
|
|
"logps/chosen": -467.75956937799043,
|
|
"logps/rejected": -351.4282877105666,
|
|
"loss": 0.5019,
|
|
"loss/base_kto": 3.1603076457977295,
|
|
"loss/total_with_kl": 4.015546798706055,
|
|
"metrics/advantage_var": 513.6570434570312,
|
|
"regularization/advantage_var": 513.6570434570312,
|
|
"regularization/kl": 0.8552390336990356,
|
|
"rewards/chosen": 0.6412341834255383,
|
|
"rewards/margins": 0.8678184149321664,
|
|
"rewards/rejected": -0.22658423150662807,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.05158233642578,
|
|
"kl": 17.881261825561523,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35184865.408805035,
|
|
"logits/rejected": -36423568.69565217,
|
|
"logps/chosen": -473.2978085691824,
|
|
"logps/rejected": -369.11201960403724,
|
|
"loss": 0.5222,
|
|
"loss/base_kto": 3.1760265827178955,
|
|
"loss/total_with_kl": 4.1773552894592285,
|
|
"metrics/advantage_var": 601.3986206054688,
|
|
"regularization/advantage_var": 601.3986206054688,
|
|
"regularization/kl": 1.0013288259506226,
|
|
"rewards/chosen": 0.5759879538098222,
|
|
"rewards/margins": 0.8547715269367632,
|
|
"rewards/rejected": -0.278783573126941,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 11.790045738220215,
|
|
"kl": 11.199904441833496,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34577200.667697065,
|
|
"logits/rejected": -35275314.95734597,
|
|
"logps/chosen": -487.47111669242656,
|
|
"logps/rejected": -350.2688586097946,
|
|
"loss": 0.5104,
|
|
"loss/base_kto": 3.13375186920166,
|
|
"loss/total_with_kl": 4.082924842834473,
|
|
"metrics/advantage_var": 570.07373046875,
|
|
"regularization/advantage_var": 570.07373046875,
|
|
"regularization/kl": 0.9491726160049438,
|
|
"rewards/chosen": 0.5863393698077666,
|
|
"rewards/margins": 0.956593428404602,
|
|
"rewards/rejected": -0.3702540585968355,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 16.616731643676758,
|
|
"kl": 10.115321159362793,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34927872.3838081,
|
|
"logits/rejected": -35150107.98042414,
|
|
"logps/chosen": -498.2335550974513,
|
|
"logps/rejected": -374.2097777324633,
|
|
"loss": 0.5363,
|
|
"loss/base_kto": 3.1613800525665283,
|
|
"loss/total_with_kl": 4.290786266326904,
|
|
"metrics/advantage_var": 678.3221435546875,
|
|
"regularization/advantage_var": 678.3221435546875,
|
|
"regularization/kl": 1.1294063329696655,
|
|
"rewards/chosen": 0.5670057122317748,
|
|
"rewards/margins": 0.9298887650888175,
|
|
"rewards/rejected": -0.3628830528570427,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 19.794078826904297,
|
|
"kl": 13.229277610778809,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35329152.580937974,
|
|
"logits/rejected": -37444599.728594504,
|
|
"logps/chosen": -499.31855143721634,
|
|
"logps/rejected": -376.1533218901454,
|
|
"loss": 0.5087,
|
|
"loss/base_kto": 3.1340301036834717,
|
|
"loss/total_with_kl": 4.0699334144592285,
|
|
"metrics/advantage_var": 562.10400390625,
|
|
"regularization/advantage_var": 562.10400390625,
|
|
"regularization/kl": 0.9359031915664673,
|
|
"rewards/chosen": 0.6185328127216102,
|
|
"rewards/margins": 0.9373207649536629,
|
|
"rewards/rejected": -0.3187879522320527,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 21.475278854370117,
|
|
"kl": 19.56073570251465,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35025939.75562701,
|
|
"logits/rejected": -36578649.48328268,
|
|
"logps/chosen": -456.02858721864953,
|
|
"logps/rejected": -377.5395611702128,
|
|
"loss": 0.5235,
|
|
"loss/base_kto": 3.1346421241760254,
|
|
"loss/total_with_kl": 4.187652587890625,
|
|
"metrics/advantage_var": 632.4390258789062,
|
|
"regularization/advantage_var": 632.4390258789062,
|
|
"regularization/kl": 1.0530109405517578,
|
|
"rewards/chosen": 0.6271888266805667,
|
|
"rewards/margins": 0.9491588764140156,
|
|
"rewards/rejected": -0.3219700497334489,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 18.9902400970459,
|
|
"kl": 19.397607803344727,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35581945.488076314,
|
|
"logits/rejected": -36827519.80337942,
|
|
"logps/chosen": -488.9942368839428,
|
|
"logps/rejected": -382.7798099078341,
|
|
"loss": 0.5211,
|
|
"loss/base_kto": 3.155086040496826,
|
|
"loss/total_with_kl": 4.168495178222656,
|
|
"metrics/advantage_var": 608.6544189453125,
|
|
"regularization/advantage_var": 608.6544189453125,
|
|
"regularization/kl": 1.0134094953536987,
|
|
"rewards/chosen": 0.6110664695169664,
|
|
"rewards/margins": 0.8886074587027744,
|
|
"rewards/rejected": -0.2775409891858079,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.5298490524292,
|
|
"kl": 18.607656478881836,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -33904899.24050633,
|
|
"logits/rejected": -36248582.32098766,
|
|
"logps/chosen": -476.12351661392404,
|
|
"logps/rejected": -395.6892361111111,
|
|
"loss": 0.5208,
|
|
"loss/base_kto": 3.171834945678711,
|
|
"loss/total_with_kl": 4.166215419769287,
|
|
"metrics/advantage_var": 597.225341796875,
|
|
"regularization/advantage_var": 597.225341796875,
|
|
"regularization/kl": 0.9943801760673523,
|
|
"rewards/chosen": 0.58079422576518,
|
|
"rewards/margins": 0.8998145323877205,
|
|
"rewards/rejected": -0.3190203066225405,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 9.113730430603027,
|
|
"kl": 20.980093002319336,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35865745.84707646,
|
|
"logits/rejected": -36366382.77324633,
|
|
"logps/chosen": -481.3038324587706,
|
|
"logps/rejected": -374.52472471451875,
|
|
"loss": 0.5078,
|
|
"loss/base_kto": 3.164994716644287,
|
|
"loss/total_with_kl": 4.062058448791504,
|
|
"metrics/advantage_var": 538.7771606445312,
|
|
"regularization/advantage_var": 538.7771606445312,
|
|
"regularization/kl": 0.8970640301704407,
|
|
"rewards/chosen": 0.6543488509651424,
|
|
"rewards/margins": 0.8510356749982977,
|
|
"rewards/rejected": -0.19668682403315532,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 14.13674259185791,
|
|
"kl": 31.113117218017578,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -37203584.9653092,
|
|
"logits/rejected": -37281410.28200972,
|
|
"logps/chosen": -460.94546568627453,
|
|
"logps/rejected": -380.5852410858995,
|
|
"loss": 0.5059,
|
|
"loss/base_kto": 3.1098976135253906,
|
|
"loss/total_with_kl": 4.047135829925537,
|
|
"metrics/advantage_var": 562.9060668945312,
|
|
"regularization/advantage_var": 562.9060668945312,
|
|
"regularization/kl": 0.9372385144233704,
|
|
"rewards/chosen": 0.8707901392227564,
|
|
"rewards/margins": 0.936078701471889,
|
|
"rewards/rejected": -0.06528856224913265,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 18.788869857788086,
|
|
"kl": 22.033191680908203,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35469781.66614664,
|
|
"logits/rejected": -36470463.499217525,
|
|
"logps/chosen": -503.5286661466459,
|
|
"logps/rejected": -389.35942390453835,
|
|
"loss": 0.5703,
|
|
"loss/base_kto": 3.1834867000579834,
|
|
"loss/total_with_kl": 4.562434196472168,
|
|
"metrics/advantage_var": 828.1964721679688,
|
|
"regularization/advantage_var": 828.1964721679688,
|
|
"regularization/kl": 1.378947138786316,
|
|
"rewards/chosen": 0.6484626853335852,
|
|
"rewards/margins": 0.8696144521628595,
|
|
"rewards/rejected": -0.22115176682927426,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 21.770421981811523,
|
|
"kl": 16.500782012939453,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34434442.325722985,
|
|
"logits/rejected": -35681284.93097913,
|
|
"logps/chosen": -509.9009703196347,
|
|
"logps/rejected": -366.6454404093098,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.1322789192199707,
|
|
"loss/total_with_kl": 4.14486837387085,
|
|
"metrics/advantage_var": 608.161865234375,
|
|
"regularization/advantage_var": 608.161865234375,
|
|
"regularization/kl": 1.012589454650879,
|
|
"rewards/chosen": 0.66641119226836,
|
|
"rewards/margins": 0.9310437831607783,
|
|
"rewards/rejected": -0.26463259089241825,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 12.632974624633789,
|
|
"kl": 18.189037322998047,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34116828.43358779,
|
|
"logits/rejected": -36532958.0032,
|
|
"logps/chosen": -456.4682251908397,
|
|
"logps/rejected": -372.2326,
|
|
"loss": 0.5149,
|
|
"loss/base_kto": 3.2032310962677,
|
|
"loss/total_with_kl": 4.119079113006592,
|
|
"metrics/advantage_var": 550.0590209960938,
|
|
"regularization/advantage_var": 550.0590209960938,
|
|
"regularization/kl": 0.9158482551574707,
|
|
"rewards/chosen": 0.6247592139790077,
|
|
"rewards/margins": 0.8383302589008828,
|
|
"rewards/rejected": -0.213571044921875,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 15.308002471923828,
|
|
"kl": 21.430456161499023,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34532757.26888218,
|
|
"logits/rejected": -36014557.20388349,
|
|
"logps/chosen": -480.0640577794562,
|
|
"logps/rejected": -388.9739583333333,
|
|
"loss": 0.5413,
|
|
"loss/base_kto": 3.1949360370635986,
|
|
"loss/total_with_kl": 4.330256938934326,
|
|
"metrics/advantage_var": 681.8742065429688,
|
|
"regularization/advantage_var": 681.8742065429688,
|
|
"regularization/kl": 1.1353205442428589,
|
|
"rewards/chosen": 0.6811915740505806,
|
|
"rewards/margins": 0.8478852028493771,
|
|
"rewards/rejected": -0.1666936287987965,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 30.371458053588867,
|
|
"kl": 22.418798446655273,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35363215.7696,
|
|
"logits/rejected": -35970881.270229004,
|
|
"logps/chosen": -465.29445,
|
|
"logps/rejected": -385.3062261450382,
|
|
"loss": 0.5415,
|
|
"loss/base_kto": 3.2369346618652344,
|
|
"loss/total_with_kl": 4.332038402557373,
|
|
"metrics/advantage_var": 657.719970703125,
|
|
"regularization/advantage_var": 657.719970703125,
|
|
"regularization/kl": 1.0951037406921387,
|
|
"rewards/chosen": 0.60460400390625,
|
|
"rewards/margins": 0.7587009305298784,
|
|
"rewards/rejected": -0.15409692662362834,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 7.786691188812256,
|
|
"kl": 19.352018356323242,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -33677738.2504065,
|
|
"logits/rejected": -36691638.47218045,
|
|
"logps/chosen": -499.4220020325203,
|
|
"logps/rejected": -400.281719924812,
|
|
"loss": 0.5055,
|
|
"loss/base_kto": 3.089252233505249,
|
|
"loss/total_with_kl": 4.043831825256348,
|
|
"metrics/advantage_var": 573.3209838867188,
|
|
"regularization/advantage_var": 573.3209838867188,
|
|
"regularization/kl": 0.9545795321464539,
|
|
"rewards/chosen": 0.6374167341526931,
|
|
"rewards/margins": 0.9257240044035154,
|
|
"rewards/rejected": -0.28830727025082237,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 23.878217697143555,
|
|
"kl": 17.17015266418457,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34343660.36691729,
|
|
"logits/rejected": -34894023.80487805,
|
|
"logps/chosen": -490.42598684210526,
|
|
"logps/rejected": -373.0741869918699,
|
|
"loss": 0.5007,
|
|
"loss/base_kto": 3.183598041534424,
|
|
"loss/total_with_kl": 4.00538969039917,
|
|
"metrics/advantage_var": 493.568603515625,
|
|
"regularization/advantage_var": 493.568603515625,
|
|
"regularization/kl": 0.8217917680740356,
|
|
"rewards/chosen": 0.6170498726063205,
|
|
"rewards/margins": 0.8689512914007819,
|
|
"rewards/rejected": -0.2519014187944614,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.530298233032227,
|
|
"kl": 23.011098861694336,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35550925.606299214,
|
|
"logits/rejected": -37183583.255813956,
|
|
"logps/chosen": -491.2183070866142,
|
|
"logps/rejected": -383.9325581395349,
|
|
"loss": 0.5324,
|
|
"loss/base_kto": 3.167588472366333,
|
|
"loss/total_with_kl": 4.259336948394775,
|
|
"metrics/advantage_var": 655.7047119140625,
|
|
"regularization/advantage_var": 655.7047119140625,
|
|
"regularization/kl": 1.0917482376098633,
|
|
"rewards/chosen": 0.7420297217181349,
|
|
"rewards/margins": 0.9214242884536908,
|
|
"rewards/rejected": -0.17939456673555595,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 10.666282653808594,
|
|
"kl": 21.266172409057617,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35279183.58346095,
|
|
"logits/rejected": -38027107.215311006,
|
|
"logps/chosen": -515.2427737366003,
|
|
"logps/rejected": -352.39478668261563,
|
|
"loss": 0.5311,
|
|
"loss/base_kto": 3.1706020832061768,
|
|
"loss/total_with_kl": 4.248837947845459,
|
|
"metrics/advantage_var": 647.5892944335938,
|
|
"regularization/advantage_var": 647.5892944335938,
|
|
"regularization/kl": 1.0782362222671509,
|
|
"rewards/chosen": 0.7221505835446975,
|
|
"rewards/margins": 0.878060266309849,
|
|
"rewards/rejected": -0.15590968276515152,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 17.214746475219727,
|
|
"kl": 20.309959411621094,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -35189824.30917875,
|
|
"logits/rejected": -37286748.84370258,
|
|
"logps/chosen": -472.0552536231884,
|
|
"logps/rejected": -365.98112670713203,
|
|
"loss": 0.509,
|
|
"loss/base_kto": 3.1659202575683594,
|
|
"loss/total_with_kl": 4.072198390960693,
|
|
"metrics/advantage_var": 544.3112182617188,
|
|
"regularization/advantage_var": 544.3112182617188,
|
|
"regularization/kl": 0.906278133392334,
|
|
"rewards/chosen": 0.6328442952867099,
|
|
"rewards/margins": 0.871725060561985,
|
|
"rewards/rejected": -0.23888076527527505,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 132.70606994628906,
|
|
"kl": 17.47706413269043,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34890082.093457945,
|
|
"logits/rejected": -35401368.47648903,
|
|
"logps/chosen": -482.0947235202492,
|
|
"logps/rejected": -382.3184267241379,
|
|
"loss": 0.52,
|
|
"loss/base_kto": 3.2115752696990967,
|
|
"loss/total_with_kl": 4.1597418785095215,
|
|
"metrics/advantage_var": 569.4691162109375,
|
|
"regularization/advantage_var": 569.4691162109375,
|
|
"regularization/kl": 0.9481660723686218,
|
|
"rewards/chosen": 0.5828254675939447,
|
|
"rewards/margins": 0.8018098263306586,
|
|
"rewards/rejected": -0.21898435873671385,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 18.174816131591797,
|
|
"kl": 10.889971733093262,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34250043.71941272,
|
|
"logits/rejected": -36426864.07196402,
|
|
"logps/chosen": -470.50881933115824,
|
|
"logps/rejected": -367.67648988006,
|
|
"loss": 0.4985,
|
|
"loss/base_kto": 3.2247750759124756,
|
|
"loss/total_with_kl": 3.9877007007598877,
|
|
"metrics/advantage_var": 458.2135314941406,
|
|
"regularization/advantage_var": 458.2135314941406,
|
|
"regularization/kl": 0.7629255652427673,
|
|
"rewards/chosen": 0.5111739141517129,
|
|
"rewards/margins": 0.8395675786475899,
|
|
"rewards/rejected": -0.32839366449587704,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 24.727251052856445,
|
|
"kl": 7.525174140930176,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33379049.296636086,
|
|
"logits/rejected": -35816722.8115016,
|
|
"logps/chosen": -473.3429854740061,
|
|
"logps/rejected": -364.55051916932905,
|
|
"loss": 0.5236,
|
|
"loss/base_kto": 3.204340696334839,
|
|
"loss/total_with_kl": 4.188536167144775,
|
|
"metrics/advantage_var": 591.1080322265625,
|
|
"regularization/advantage_var": 591.1080322265625,
|
|
"regularization/kl": 0.9841948747634888,
|
|
"rewards/chosen": 0.44478586179400803,
|
|
"rewards/margins": 0.8876847431151442,
|
|
"rewards/rejected": -0.4428988813211362,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 81.66817474365234,
|
|
"kl": 19.855487823486328,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35810387.82664526,
|
|
"logits/rejected": -35341082.88584475,
|
|
"logps/chosen": -499.32609349919744,
|
|
"logps/rejected": -395.1006230974125,
|
|
"loss": 0.5183,
|
|
"loss/base_kto": 3.1676383018493652,
|
|
"loss/total_with_kl": 4.1465253829956055,
|
|
"metrics/advantage_var": 587.9201049804688,
|
|
"regularization/advantage_var": 587.9201049804688,
|
|
"regularization/kl": 0.9788870215415955,
|
|
"rewards/chosen": 0.6302710666319723,
|
|
"rewards/margins": 0.8804101840543229,
|
|
"rewards/rejected": -0.2501391174223506,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 13.090487480163574,
|
|
"kl": 11.669354438781738,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33785616.0,
|
|
"logits/rejected": -36079353.6,
|
|
"logps/chosen": -470.088134765625,
|
|
"logps/rejected": -378.515869140625,
|
|
"loss": 0.5388,
|
|
"loss/base_kto": 3.1741302013397217,
|
|
"loss/total_with_kl": 4.31029748916626,
|
|
"metrics/advantage_var": 682.3826904296875,
|
|
"regularization/advantage_var": 682.3826904296875,
|
|
"regularization/kl": 1.1361671686172485,
|
|
"rewards/chosen": 0.55972900390625,
|
|
"rewards/margins": 0.91335186958313,
|
|
"rewards/rejected": -0.3536228656768799,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 11.46224594116211,
|
|
"kl": 14.195319175720215,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -35331752.10794602,
|
|
"logits/rejected": -35831473.90538336,
|
|
"logps/chosen": -469.05406671664167,
|
|
"logps/rejected": -360.18046492659056,
|
|
"loss": 0.6604,
|
|
"loss/base_kto": 3.2137787342071533,
|
|
"loss/total_with_kl": 5.2835774421691895,
|
|
"metrics/advantage_var": 1243.1224365234375,
|
|
"regularization/advantage_var": 1243.1224365234375,
|
|
"regularization/kl": 2.0697989463806152,
|
|
"rewards/chosen": 0.6094958349682581,
|
|
"rewards/margins": 0.8998844805573181,
|
|
"rewards/rejected": -0.29038864558905997,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 10.405191421508789,
|
|
"kl": 19.894010543823242,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34590439.9872814,
|
|
"logits/rejected": -36019840.19662058,
|
|
"logps/chosen": -495.714626391097,
|
|
"logps/rejected": -385.013176843318,
|
|
"loss": 0.4964,
|
|
"loss/base_kto": 3.168362617492676,
|
|
"loss/total_with_kl": 3.9712867736816406,
|
|
"metrics/advantage_var": 482.2367858886719,
|
|
"regularization/advantage_var": 482.2367858886719,
|
|
"regularization/kl": 0.8029242753982544,
|
|
"rewards/chosen": 0.6687352312388215,
|
|
"rewards/margins": 0.8462016398702239,
|
|
"rewards/rejected": -0.17746640863140242,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 13.125397682189941,
|
|
"kl": 15.497633934020996,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35739088.87363494,
|
|
"logits/rejected": -36400065.50234742,
|
|
"logps/chosen": -479.00409516380654,
|
|
"logps/rejected": -362.5519610719875,
|
|
"loss": 0.5092,
|
|
"loss/base_kto": 3.217585802078247,
|
|
"loss/total_with_kl": 4.0731987953186035,
|
|
"metrics/advantage_var": 513.8816528320312,
|
|
"regularization/advantage_var": 513.8816528320312,
|
|
"regularization/kl": 0.8556130528450012,
|
|
"rewards/chosen": 0.5395711580416341,
|
|
"rewards/margins": 0.808753891568446,
|
|
"rewards/rejected": -0.26918273352681193,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 9.873025894165039,
|
|
"kl": 12.705950736999512,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34435339.41401274,
|
|
"logits/rejected": -36264476.26993865,
|
|
"logps/chosen": -465.34633757961785,
|
|
"logps/rejected": -391.04637174079755,
|
|
"loss": 0.5051,
|
|
"loss/base_kto": 3.231473207473755,
|
|
"loss/total_with_kl": 4.040536403656006,
|
|
"metrics/advantage_var": 485.9240417480469,
|
|
"regularization/advantage_var": 485.9240417480469,
|
|
"regularization/kl": 0.8090634346008301,
|
|
"rewards/chosen": 0.4578510940454568,
|
|
"rewards/margins": 0.7875057174398203,
|
|
"rewards/rejected": -0.3296546233943635,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 19.27297592163086,
|
|
"kl": 13.081128120422363,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34923876.243369736,
|
|
"logits/rejected": -37735089.076682314,
|
|
"logps/chosen": -486.90103354134163,
|
|
"logps/rejected": -390.3783499608764,
|
|
"loss": 0.5035,
|
|
"loss/base_kto": 3.196063756942749,
|
|
"loss/total_with_kl": 4.028075695037842,
|
|
"metrics/advantage_var": 499.7066345214844,
|
|
"regularization/advantage_var": 499.7066345214844,
|
|
"regularization/kl": 0.8320116400718689,
|
|
"rewards/chosen": 0.5644023258488934,
|
|
"rewards/margins": 0.8339658604855327,
|
|
"rewards/rejected": -0.2695635346366393,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 47.9726448059082,
|
|
"kl": 14.978861808776855,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33340049.265116278,
|
|
"logits/rejected": -35089638.6015748,
|
|
"logps/chosen": -504.12107558139536,
|
|
"logps/rejected": -365.18033956692915,
|
|
"loss": 0.5166,
|
|
"loss/base_kto": 3.168360471725464,
|
|
"loss/total_with_kl": 4.132617473602295,
|
|
"metrics/advantage_var": 579.13330078125,
|
|
"regularization/advantage_var": 579.13330078125,
|
|
"regularization/kl": 0.9642569422721863,
|
|
"rewards/chosen": 0.5865663513656735,
|
|
"rewards/margins": 0.9104793969334639,
|
|
"rewards/rejected": -0.3239130455677904,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 12.994561195373535,
|
|
"kl": 20.298545837402344,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33154546.52631579,
|
|
"logits/rejected": -34941696.0,
|
|
"logps/chosen": -489.10043174342104,
|
|
"logps/rejected": -368.0780784970238,
|
|
"loss": 0.5073,
|
|
"loss/base_kto": 3.1604514122009277,
|
|
"loss/total_with_kl": 4.058101177215576,
|
|
"metrics/advantage_var": 539.12890625,
|
|
"regularization/advantage_var": 539.12890625,
|
|
"regularization/kl": 0.8976497054100037,
|
|
"rewards/chosen": 0.6708749469957853,
|
|
"rewards/margins": 0.889276719631109,
|
|
"rewards/rejected": -0.21840177263532365,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 6.703609943389893,
|
|
"kl": 10.383610725402832,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35828997.274167985,
|
|
"logits/rejected": -36020733.625966,
|
|
"logps/chosen": -499.90887480190173,
|
|
"logps/rejected": -372.4459524729521,
|
|
"loss": 0.4814,
|
|
"loss/base_kto": 3.183142900466919,
|
|
"loss/total_with_kl": 3.851154327392578,
|
|
"metrics/advantage_var": 401.2080993652344,
|
|
"regularization/advantage_var": 401.2080993652344,
|
|
"regularization/kl": 0.668011486530304,
|
|
"rewards/chosen": 0.4524446176084588,
|
|
"rewards/margins": 0.8381700841170441,
|
|
"rewards/rejected": -0.3857254665085853,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 20.486181259155273,
|
|
"kl": 12.683451652526855,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34344392.36535433,
|
|
"logits/rejected": -35632847.18139535,
|
|
"logps/chosen": -482.1603346456693,
|
|
"logps/rejected": -389.7322189922481,
|
|
"loss": 0.4673,
|
|
"loss/base_kto": 3.0994527339935303,
|
|
"loss/total_with_kl": 3.7386157512664795,
|
|
"metrics/advantage_var": 383.8816833496094,
|
|
"regularization/advantage_var": 383.8816833496094,
|
|
"regularization/kl": 0.6391629576683044,
|
|
"rewards/chosen": 0.5894631213090551,
|
|
"rewards/margins": 0.9362109410764969,
|
|
"rewards/rejected": -0.34674781976744184,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 12.909422874450684,
|
|
"kl": 12.25561809539795,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34363460.266666666,
|
|
"logits/rejected": -34902749.31612903,
|
|
"logps/chosen": -479.07670454545456,
|
|
"logps/rejected": -381.37142137096777,
|
|
"loss": 0.4675,
|
|
"loss/base_kto": 3.1592354774475098,
|
|
"loss/total_with_kl": 3.7402336597442627,
|
|
"metrics/advantage_var": 348.9477233886719,
|
|
"regularization/advantage_var": 348.9477233886719,
|
|
"regularization/kl": 0.580997884273529,
|
|
"rewards/chosen": 0.5304400819720644,
|
|
"rewards/margins": 0.8264333208518526,
|
|
"rewards/rejected": -0.2959932388797883,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 8.274309158325195,
|
|
"kl": 16.63254165649414,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35039798.327974275,
|
|
"logits/rejected": -37158880.87537994,
|
|
"logps/chosen": -479.2469855305466,
|
|
"logps/rejected": -405.74534574468083,
|
|
"loss": 0.4626,
|
|
"loss/base_kto": 3.1412341594696045,
|
|
"loss/total_with_kl": 3.700852632522583,
|
|
"metrics/advantage_var": 336.1070251464844,
|
|
"regularization/advantage_var": 336.1070251464844,
|
|
"regularization/kl": 0.5596182346343994,
|
|
"rewards/chosen": 0.5669709723862038,
|
|
"rewards/margins": 0.8345602747970368,
|
|
"rewards/rejected": -0.267589302410833,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 13.507829666137695,
|
|
"kl": 15.458251953125,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34374997.333333336,
|
|
"logits/rejected": -35833673.884555385,
|
|
"logps/chosen": -454.6964495305164,
|
|
"logps/rejected": -376.2020768330733,
|
|
"loss": 0.46,
|
|
"loss/base_kto": 3.1337621212005615,
|
|
"loss/total_with_kl": 3.6797828674316406,
|
|
"metrics/advantage_var": 327.9403991699219,
|
|
"regularization/advantage_var": 327.9403991699219,
|
|
"regularization/kl": 0.5460208058357239,
|
|
"rewards/chosen": 0.5840901119608275,
|
|
"rewards/margins": 0.86935336392157,
|
|
"rewards/rejected": -0.2852632519607425,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 9.739164352416992,
|
|
"kl": 12.5602388381958,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -36261101.4939759,
|
|
"logits/rejected": -36452877.2987013,
|
|
"logps/chosen": -475.23870481927713,
|
|
"logps/rejected": -375.8420505275974,
|
|
"loss": 0.4679,
|
|
"loss/base_kto": 3.14152193069458,
|
|
"loss/total_with_kl": 3.743072509765625,
|
|
"metrics/advantage_var": 361.2915954589844,
|
|
"regularization/advantage_var": 361.2915954589844,
|
|
"regularization/kl": 0.6015505194664001,
|
|
"rewards/chosen": 0.6037944196218468,
|
|
"rewards/margins": 0.8639737658798368,
|
|
"rewards/rejected": -0.26017934625799005,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 11.08957576751709,
|
|
"kl": 16.796659469604492,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34987716.678071536,
|
|
"logits/rejected": -36955770.97645212,
|
|
"logps/chosen": -497.6564930015552,
|
|
"logps/rejected": -363.1662333202512,
|
|
"loss": 0.47,
|
|
"loss/base_kto": 3.0917279720306396,
|
|
"loss/total_with_kl": 3.7601680755615234,
|
|
"metrics/advantage_var": 401.4655456542969,
|
|
"regularization/advantage_var": 401.4655456542969,
|
|
"regularization/kl": 0.6684401631355286,
|
|
"rewards/chosen": 0.6684237134586412,
|
|
"rewards/margins": 0.9006318935493373,
|
|
"rewards/rejected": -0.23220818009069613,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 12.405346870422363,
|
|
"kl": 13.004144668579102,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35085379.62264151,
|
|
"logits/rejected": -36464627.27950311,
|
|
"logps/chosen": -478.507419418239,
|
|
"logps/rejected": -370.686286878882,
|
|
"loss": 0.4676,
|
|
"loss/base_kto": 3.133150100708008,
|
|
"loss/total_with_kl": 3.7407402992248535,
|
|
"metrics/advantage_var": 364.9190368652344,
|
|
"regularization/advantage_var": 364.9190368652344,
|
|
"regularization/kl": 0.6075901389122009,
|
|
"rewards/chosen": 0.5498898104301788,
|
|
"rewards/margins": 0.8730859135439575,
|
|
"rewards/rejected": -0.3231961031137786,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 18.598962783813477,
|
|
"kl": 18.174997329711914,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34538869.84126984,
|
|
"logits/rejected": -34755177.55076923,
|
|
"logps/chosen": -465.79578373015875,
|
|
"logps/rejected": -381.8952884615385,
|
|
"loss": 0.4713,
|
|
"loss/base_kto": 3.144207000732422,
|
|
"loss/total_with_kl": 3.7700016498565674,
|
|
"metrics/advantage_var": 375.8526306152344,
|
|
"regularization/advantage_var": 375.8526306152344,
|
|
"regularization/kl": 0.6257945895195007,
|
|
"rewards/chosen": 0.5810432070777529,
|
|
"rewards/margins": 0.8297481349623683,
|
|
"rewards/rejected": -0.24870492788461537,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 8.991421699523926,
|
|
"kl": 13.875744819641113,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35983386.256410256,
|
|
"logits/rejected": -37287817.36585366,
|
|
"logps/chosen": -476.5966546474359,
|
|
"logps/rejected": -367.7307069359756,
|
|
"loss": 0.4629,
|
|
"loss/base_kto": 3.1394855976104736,
|
|
"loss/total_with_kl": 3.703155517578125,
|
|
"metrics/advantage_var": 338.5404357910156,
|
|
"regularization/advantage_var": 338.5404357910156,
|
|
"regularization/kl": 0.5636698603630066,
|
|
"rewards/chosen": 0.5675454261975411,
|
|
"rewards/margins": 0.8820818232476078,
|
|
"rewards/rejected": -0.3145363970500667,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 14.043456077575684,
|
|
"kl": 11.75983715057373,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34603116.27794562,
|
|
"logits/rejected": -36045151.275080904,
|
|
"logps/chosen": -487.02841767371604,
|
|
"logps/rejected": -367.93805622977345,
|
|
"loss": 0.4656,
|
|
"loss/base_kto": 3.124924421310425,
|
|
"loss/total_with_kl": 3.724797010421753,
|
|
"metrics/advantage_var": 360.28399658203125,
|
|
"regularization/advantage_var": 360.28399658203125,
|
|
"regularization/kl": 0.599872887134552,
|
|
"rewards/chosen": 0.5678141614101444,
|
|
"rewards/margins": 0.8667035437817692,
|
|
"rewards/rejected": -0.2988893823716247,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 11.319145202636719,
|
|
"kl": 20.254621505737305,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34745140.49211357,
|
|
"logits/rejected": -35675890.526315786,
|
|
"logps/chosen": -484.886238170347,
|
|
"logps/rejected": -388.7073335913313,
|
|
"loss": 0.4664,
|
|
"loss/base_kto": 3.105311155319214,
|
|
"loss/total_with_kl": 3.7308807373046875,
|
|
"metrics/advantage_var": 375.7174987792969,
|
|
"regularization/advantage_var": 375.7174987792969,
|
|
"regularization/kl": 0.6255696415901184,
|
|
"rewards/chosen": 0.7004149524195337,
|
|
"rewards/margins": 0.894981023637546,
|
|
"rewards/rejected": -0.19456607121801228,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 6.997559070587158,
|
|
"kl": 13.4202880859375,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34595846.28220859,
|
|
"logits/rejected": -36061069.85987261,
|
|
"logps/chosen": -465.620446702454,
|
|
"logps/rejected": -370.55667794585986,
|
|
"loss": 0.4606,
|
|
"loss/base_kto": 3.147129535675049,
|
|
"loss/total_with_kl": 3.6847634315490723,
|
|
"metrics/advantage_var": 322.9032287597656,
|
|
"regularization/advantage_var": 322.9032287597656,
|
|
"regularization/kl": 0.5376338362693787,
|
|
"rewards/chosen": 0.5557675507902368,
|
|
"rewards/margins": 0.8490083834652715,
|
|
"rewards/rejected": -0.2932408326750348,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 12.055425643920898,
|
|
"kl": 14.774609565734863,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35488962.97451274,
|
|
"logits/rejected": -36899754.80587275,
|
|
"logps/chosen": -524.7817185157421,
|
|
"logps/rejected": -358.01661908646,
|
|
"loss": 0.4679,
|
|
"loss/base_kto": 3.0890705585479736,
|
|
"loss/total_with_kl": 3.743443727493286,
|
|
"metrics/advantage_var": 393.0169372558594,
|
|
"regularization/advantage_var": 393.0169372558594,
|
|
"regularization/kl": 0.6543732285499573,
|
|
"rewards/chosen": 0.6401997213063391,
|
|
"rewards/margins": 0.9174135849093927,
|
|
"rewards/rejected": -0.2772138636030536,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 8.9901704788208,
|
|
"kl": 16.92176628112793,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34209161.35614308,
|
|
"logits/rejected": -36074195.39089482,
|
|
"logps/chosen": -491.05350894245726,
|
|
"logps/rejected": -366.507751177394,
|
|
"loss": 0.4678,
|
|
"loss/base_kto": 3.1431918144226074,
|
|
"loss/total_with_kl": 3.742206335067749,
|
|
"metrics/advantage_var": 359.76837158203125,
|
|
"regularization/advantage_var": 359.76837158203125,
|
|
"regularization/kl": 0.5990143418312073,
|
|
"rewards/chosen": 0.5929558785113724,
|
|
"rewards/margins": 0.846540089013482,
|
|
"rewards/rejected": -0.2535842105021095,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 7.918067932128906,
|
|
"kl": 19.62078857421875,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34853530.33492823,
|
|
"logits/rejected": -35989707.85911179,
|
|
"logps/chosen": -449.56539074960125,
|
|
"logps/rejected": -385.4023258039816,
|
|
"loss": 0.4543,
|
|
"loss/base_kto": 3.0972354412078857,
|
|
"loss/total_with_kl": 3.634321689605713,
|
|
"metrics/advantage_var": 322.5742492675781,
|
|
"regularization/advantage_var": 322.5742492675781,
|
|
"regularization/kl": 0.5370860695838928,
|
|
"rewards/chosen": 0.6514162979247657,
|
|
"rewards/margins": 0.8836477768860502,
|
|
"rewards/rejected": -0.23223147896128446,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 10.17796802520752,
|
|
"kl": 15.71185302734375,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33575923.39692308,
|
|
"logits/rejected": -36114002.895238094,
|
|
"logps/chosen": -475.3112980769231,
|
|
"logps/rejected": -370.634375,
|
|
"loss": 0.4581,
|
|
"loss/base_kto": 3.1170105934143066,
|
|
"loss/total_with_kl": 3.6648223400115967,
|
|
"metrics/advantage_var": 329.01629638671875,
|
|
"regularization/advantage_var": 329.01629638671875,
|
|
"regularization/kl": 0.5478121042251587,
|
|
"rewards/chosen": 0.6221265587439904,
|
|
"rewards/margins": 0.8973633121774601,
|
|
"rewards/rejected": -0.27523675343346976,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 9.963101387023926,
|
|
"kl": 15.151724815368652,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34225013.89473684,
|
|
"logits/rejected": -36751024.76190476,
|
|
"logps/chosen": -474.8861019736842,
|
|
"logps/rejected": -376.24004836309524,
|
|
"loss": 0.4609,
|
|
"loss/base_kto": 3.1314008235931396,
|
|
"loss/total_with_kl": 3.6872289180755615,
|
|
"metrics/advantage_var": 333.8307189941406,
|
|
"regularization/advantage_var": 333.8307189941406,
|
|
"regularization/kl": 0.5558281540870667,
|
|
"rewards/chosen": 0.5898361707988539,
|
|
"rewards/margins": 0.8616058306586474,
|
|
"rewards/rejected": -0.2717696598597935,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 8.92536449432373,
|
|
"kl": 13.524930000305176,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34468356.72615384,
|
|
"logits/rejected": -36522699.17460317,
|
|
"logps/chosen": -478.20240384615386,
|
|
"logps/rejected": -361.2171626984127,
|
|
"loss": 0.4656,
|
|
"loss/base_kto": 3.1335270404815674,
|
|
"loss/total_with_kl": 3.724987030029297,
|
|
"metrics/advantage_var": 355.2311096191406,
|
|
"regularization/advantage_var": 355.2311096191406,
|
|
"regularization/kl": 0.5914598107337952,
|
|
"rewards/chosen": 0.5554096867487981,
|
|
"rewards/margins": 0.8794731846251622,
|
|
"rewards/rejected": -0.32406349787636407,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 10.868731498718262,
|
|
"kl": 13.2905912399292,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34153105.94015748,
|
|
"logits/rejected": -35693204.440310076,
|
|
"logps/chosen": -508.6543799212598,
|
|
"logps/rejected": -383.42766472868215,
|
|
"loss": 0.4592,
|
|
"loss/base_kto": 3.1157286167144775,
|
|
"loss/total_with_kl": 3.6738109588623047,
|
|
"metrics/advantage_var": 335.1844787597656,
|
|
"regularization/advantage_var": 335.1844787597656,
|
|
"regularization/kl": 0.5580821633338928,
|
|
"rewards/chosen": 0.5697481891301673,
|
|
"rewards/margins": 0.8933302336281082,
|
|
"rewards/rejected": -0.32358204449794087,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.145153045654297,
|
|
"kl": 15.356244087219238,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34969000.3456,
|
|
"logits/rejected": -36647656.15877862,
|
|
"logps/chosen": -460.0201,
|
|
"logps/rejected": -378.9165791984733,
|
|
"loss": 0.4654,
|
|
"loss/base_kto": 3.089923620223999,
|
|
"loss/total_with_kl": 3.7231736183166504,
|
|
"metrics/advantage_var": 380.330322265625,
|
|
"regularization/advantage_var": 380.330322265625,
|
|
"regularization/kl": 0.6332499384880066,
|
|
"rewards/chosen": 0.59560576171875,
|
|
"rewards/margins": 0.9119439412124284,
|
|
"rewards/rejected": -0.31633817949367843,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 18.175031661987305,
|
|
"kl": 13.56359577178955,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35729968.83692308,
|
|
"logits/rejected": -36075240.43174603,
|
|
"logps/chosen": -483.73841346153847,
|
|
"logps/rejected": -394.5961557539683,
|
|
"loss": 0.4727,
|
|
"loss/base_kto": 3.1271884441375732,
|
|
"loss/total_with_kl": 3.7812340259552,
|
|
"metrics/advantage_var": 392.8203430175781,
|
|
"regularization/advantage_var": 392.8203430175781,
|
|
"regularization/kl": 0.654045820236206,
|
|
"rewards/chosen": 0.5983189978966346,
|
|
"rewards/margins": 0.891011248950702,
|
|
"rewards/rejected": -0.2926922510540675,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 13.319075584411621,
|
|
"kl": 15.165995597839355,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34791957.82295082,
|
|
"logits/rejected": -36172347.60597015,
|
|
"logps/chosen": -474.82899590163936,
|
|
"logps/rejected": -401.5444962686567,
|
|
"loss": 0.4701,
|
|
"loss/base_kto": 3.161012649536133,
|
|
"loss/total_with_kl": 3.761154890060425,
|
|
"metrics/advantage_var": 360.4458923339844,
|
|
"regularization/advantage_var": 360.4458923339844,
|
|
"regularization/kl": 0.6001423597335815,
|
|
"rewards/chosen": 0.5497582607581967,
|
|
"rewards/margins": 0.8475414775477839,
|
|
"rewards/rejected": -0.29778321678958725,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 15.17506217956543,
|
|
"kl": 9.989168167114258,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35060965.54531001,
|
|
"logits/rejected": -35499975.37327189,
|
|
"logps/chosen": -507.5405405405405,
|
|
"logps/rejected": -360.3473742319508,
|
|
"loss": 0.4705,
|
|
"loss/base_kto": 3.1441006660461426,
|
|
"loss/total_with_kl": 3.76410174369812,
|
|
"metrics/advantage_var": 372.3731384277344,
|
|
"regularization/advantage_var": 372.3731384277344,
|
|
"regularization/kl": 0.6200013160705566,
|
|
"rewards/chosen": 0.5026958326088037,
|
|
"rewards/margins": 0.8739539134384768,
|
|
"rewards/rejected": -0.3712580808296731,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 10.091706275939941,
|
|
"kl": 12.291145324707031,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35119788.8,
|
|
"logits/rejected": -35423200.0,
|
|
"logps/chosen": -499.049609375,
|
|
"logps/rejected": -407.73125,
|
|
"loss": 0.464,
|
|
"loss/base_kto": 3.096036911010742,
|
|
"loss/total_with_kl": 3.7123208045959473,
|
|
"metrics/advantage_var": 370.140380859375,
|
|
"regularization/advantage_var": 370.140380859375,
|
|
"regularization/kl": 0.6162838339805603,
|
|
"rewards/chosen": 0.5213249683380127,
|
|
"rewards/margins": 0.9092860221862793,
|
|
"rewards/rejected": -0.3879610538482666,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 12.230925559997559,
|
|
"kl": 11.54342269897461,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -35085299.2,
|
|
"logits/rejected": -36306348.8,
|
|
"logps/chosen": -498.4134765625,
|
|
"logps/rejected": -377.816064453125,
|
|
"loss": 0.4591,
|
|
"loss/base_kto": 3.092241048812866,
|
|
"loss/total_with_kl": 3.672832489013672,
|
|
"metrics/advantage_var": 348.70391845703125,
|
|
"regularization/advantage_var": 348.70391845703125,
|
|
"regularization/kl": 0.5805919766426086,
|
|
"rewards/chosen": 0.5286848068237304,
|
|
"rewards/margins": 0.9156812667846679,
|
|
"rewards/rejected": -0.3869964599609375,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 10.809049606323242,
|
|
"kl": 9.5625,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34057415.98825257,
|
|
"logits/rejected": -36385041.52253756,
|
|
"logps/chosen": -503.7184287812041,
|
|
"logps/rejected": -352.2060204507512,
|
|
"loss": 0.492,
|
|
"loss/base_kto": 3.1409976482391357,
|
|
"loss/total_with_kl": 3.936307907104492,
|
|
"metrics/advantage_var": 477.6637268066406,
|
|
"regularization/advantage_var": 477.6637268066406,
|
|
"regularization/kl": 0.7953101396560669,
|
|
"rewards/chosen": 0.5560439861818557,
|
|
"rewards/margins": 0.9007480808154898,
|
|
"rewards/rejected": -0.34470409463363416,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 9.063849449157715,
|
|
"kl": 15.310641288757324,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33109751.116719242,
|
|
"logits/rejected": -36221435.24458204,
|
|
"logps/chosen": -474.403785488959,
|
|
"logps/rejected": -352.32580301857587,
|
|
"loss": 0.4633,
|
|
"loss/base_kto": 3.134089708328247,
|
|
"loss/total_with_kl": 3.7060189247131348,
|
|
"metrics/advantage_var": 343.5010681152344,
|
|
"regularization/advantage_var": 343.5010681152344,
|
|
"regularization/kl": 0.5719292759895325,
|
|
"rewards/chosen": 0.5670358555550079,
|
|
"rewards/margins": 0.8584075347134106,
|
|
"rewards/rejected": -0.2913716791584027,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 10.3336763381958,
|
|
"kl": 17.177276611328125,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34365693.99058085,
|
|
"logits/rejected": -37128803.53343701,
|
|
"logps/chosen": -492.1367739403454,
|
|
"logps/rejected": -375.41927488335926,
|
|
"loss": 0.456,
|
|
"loss/base_kto": 3.0802464485168457,
|
|
"loss/total_with_kl": 3.647966146469116,
|
|
"metrics/advantage_var": 340.9729919433594,
|
|
"regularization/advantage_var": 340.9729919433594,
|
|
"regularization/kl": 0.5677200555801392,
|
|
"rewards/chosen": 0.6075130444680877,
|
|
"rewards/margins": 0.914466249202973,
|
|
"rewards/rejected": -0.3069532047348853,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 12.794276237487793,
|
|
"kl": 12.41401195526123,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35897003.71692308,
|
|
"logits/rejected": -37448583.72063492,
|
|
"logps/chosen": -484.5631730769231,
|
|
"logps/rejected": -380.8362599206349,
|
|
"loss": 0.46,
|
|
"loss/base_kto": 3.1174824237823486,
|
|
"loss/total_with_kl": 3.679964065551758,
|
|
"metrics/advantage_var": 337.826904296875,
|
|
"regularization/advantage_var": 337.826904296875,
|
|
"regularization/kl": 0.5624818205833435,
|
|
"rewards/chosen": 0.6082245342548077,
|
|
"rewards/margins": 0.9049272552602021,
|
|
"rewards/rejected": -0.29670272100539435,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 17.76041603088379,
|
|
"kl": 16.423965454101562,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34931012.05063291,
|
|
"logits/rejected": -35125466.074074075,
|
|
"logps/chosen": -482.17513844936707,
|
|
"logps/rejected": -353.14067322530866,
|
|
"loss": 0.4694,
|
|
"loss/base_kto": 3.076963186264038,
|
|
"loss/total_with_kl": 3.7554919719696045,
|
|
"metrics/advantage_var": 407.5247497558594,
|
|
"regularization/advantage_var": 407.5247497558594,
|
|
"regularization/kl": 0.6785287261009216,
|
|
"rewards/chosen": 0.6269692529605914,
|
|
"rewards/margins": 0.9440071885707622,
|
|
"rewards/rejected": -0.31703793561017074,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 9.461185455322266,
|
|
"kl": 15.859339714050293,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35878449.23076923,
|
|
"logits/rejected": -35323769.75609756,
|
|
"logps/chosen": -459.3909254807692,
|
|
"logps/rejected": -403.8202648628049,
|
|
"loss": 0.464,
|
|
"loss/base_kto": 3.079575300216675,
|
|
"loss/total_with_kl": 3.712149143218994,
|
|
"metrics/advantage_var": 379.92425537109375,
|
|
"regularization/advantage_var": 379.92425537109375,
|
|
"regularization/kl": 0.6325739026069641,
|
|
"rewards/chosen": 0.6018277192727114,
|
|
"rewards/margins": 0.9291157188677952,
|
|
"rewards/rejected": -0.32728799959508387,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 11.36967945098877,
|
|
"kl": 12.651641845703125,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34188158.197183095,
|
|
"logits/rejected": -35288410.65834633,
|
|
"logps/chosen": -464.17233959311426,
|
|
"logps/rejected": -374.37760822932916,
|
|
"loss": 0.4644,
|
|
"loss/base_kto": 3.135221481323242,
|
|
"loss/total_with_kl": 3.715381622314453,
|
|
"metrics/advantage_var": 348.4446716308594,
|
|
"regularization/advantage_var": 348.4446716308594,
|
|
"regularization/kl": 0.58016037940979,
|
|
"rewards/chosen": 0.5999769231709703,
|
|
"rewards/margins": 0.8528162481697514,
|
|
"rewards/rejected": -0.2528393249987812,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 11.526399612426758,
|
|
"kl": 14.372525215148926,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35875684.17391305,
|
|
"logits/rejected": -36564476.77987421,
|
|
"logps/chosen": -496.04580745341616,
|
|
"logps/rejected": -369.88600628930817,
|
|
"loss": 0.4594,
|
|
"loss/base_kto": 3.0919206142425537,
|
|
"loss/total_with_kl": 3.6748077869415283,
|
|
"metrics/advantage_var": 350.0826416015625,
|
|
"regularization/advantage_var": 350.0826416015625,
|
|
"regularization/kl": 0.582887589931488,
|
|
"rewards/chosen": 0.5898574923876649,
|
|
"rewards/margins": 0.9130547192419053,
|
|
"rewards/rejected": -0.32319722685424035,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 8.888420104980469,
|
|
"kl": 15.188580513000488,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35071442.91823899,
|
|
"logits/rejected": -36805921.39130435,
|
|
"logps/chosen": -470.28911163522014,
|
|
"logps/rejected": -371.9250776397516,
|
|
"loss": 0.4576,
|
|
"loss/base_kto": 3.078230619430542,
|
|
"loss/total_with_kl": 3.6605050563812256,
|
|
"metrics/advantage_var": 349.7142333984375,
|
|
"regularization/advantage_var": 349.7142333984375,
|
|
"regularization/kl": 0.5822741985321045,
|
|
"rewards/chosen": 0.59386304939318,
|
|
"rewards/margins": 0.9161499514002889,
|
|
"rewards/rejected": -0.3222869020071089,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 13.013932228088379,
|
|
"kl": 15.081308364868164,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35560366.82926829,
|
|
"logits/rejected": -35769974.15384615,
|
|
"logps/chosen": -469.3050209603659,
|
|
"logps/rejected": -389.21817407852564,
|
|
"loss": 0.4646,
|
|
"loss/base_kto": 3.158924102783203,
|
|
"loss/total_with_kl": 3.7165520191192627,
|
|
"metrics/advantage_var": 334.9115905761719,
|
|
"regularization/advantage_var": 334.9115905761719,
|
|
"regularization/kl": 0.55762779712677,
|
|
"rewards/chosen": 0.5677999170815072,
|
|
"rewards/margins": 0.8535911674571082,
|
|
"rewards/rejected": -0.28579125037560094,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.753754138946533,
|
|
"kl": 13.397297859191895,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34964381.3206951,
|
|
"logits/rejected": -37000560.766615145,
|
|
"logps/chosen": -477.7504443127962,
|
|
"logps/rejected": -362.28767387944356,
|
|
"loss": 0.4592,
|
|
"loss/base_kto": 3.149555206298828,
|
|
"loss/total_with_kl": 3.6733086109161377,
|
|
"metrics/advantage_var": 314.5665588378906,
|
|
"regularization/advantage_var": 314.5665588378906,
|
|
"regularization/kl": 0.5237533450126648,
|
|
"rewards/chosen": 0.5751637824903733,
|
|
"rewards/margins": 0.8500131096131515,
|
|
"rewards/rejected": -0.2748493271227782,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 9.024127960205078,
|
|
"kl": 14.589932441711426,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33744376.788732395,
|
|
"logits/rejected": -35623889.67238689,
|
|
"logps/chosen": -474.76951291079814,
|
|
"logps/rejected": -351.01794071762873,
|
|
"loss": 0.4655,
|
|
"loss/base_kto": 3.133312940597534,
|
|
"loss/total_with_kl": 3.7237255573272705,
|
|
"metrics/advantage_var": 354.6022033691406,
|
|
"regularization/advantage_var": 354.6022033691406,
|
|
"regularization/kl": 0.5904126167297363,
|
|
"rewards/chosen": 0.5715717918622848,
|
|
"rewards/margins": 0.8627933271983076,
|
|
"rewards/rejected": -0.2912215353360228,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_loss": 0.5023879745463633,
|
|
"train_runtime": 11027.9329,
|
|
"train_samples_per_second": 13.44,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|