e159cbdd46
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed1 Source: Original Platform
2229 lines
80 KiB
JSON
2229 lines
80 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 15.313405990600586,
|
|
"kl": 6.287380218505859,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36423765.73023256,
|
|
"logits/rejected": -36855343.571653545,
|
|
"logps/chosen": -500.0248546511628,
|
|
"logps/rejected": -362.0871062992126,
|
|
"loss": 0.5883,
|
|
"loss/base_kto": 3.9272773265838623,
|
|
"metrics/advantage_var": 169.6308135986328,
|
|
"regularization/mmd": 0.7794427871704102,
|
|
"regularization/rkhs_norm": 150.18167114257812,
|
|
"rewards/chosen": 0.057254383175872094,
|
|
"rewards/margins": 0.06401568852682905,
|
|
"rewards/rejected": -0.006761305350956954,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 19.27733039855957,
|
|
"kl": 8.419197082519531,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35511164.47706422,
|
|
"logits/rejected": -37369142.79872204,
|
|
"logps/chosen": -485.55103211009174,
|
|
"logps/rejected": -375.57071186102235,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.922342300415039,
|
|
"metrics/advantage_var": 186.844970703125,
|
|
"regularization/mmd": 0.798768937587738,
|
|
"regularization/rkhs_norm": 153.9053955078125,
|
|
"rewards/chosen": 0.09748388284572403,
|
|
"rewards/margins": 0.0519025923294953,
|
|
"rewards/rejected": 0.04558129051622872,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 18.518787384033203,
|
|
"kl": 14.31860065460205,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36978193.22324159,
|
|
"logits/rejected": -37764125.44408946,
|
|
"logps/chosen": -468.082377675841,
|
|
"logps/rejected": -352.98507388178916,
|
|
"loss": 0.579,
|
|
"loss/base_kto": 3.8540725708007812,
|
|
"metrics/advantage_var": 169.3621826171875,
|
|
"regularization/mmd": 0.7777101397514343,
|
|
"regularization/rkhs_norm": 149.84780883789062,
|
|
"rewards/chosen": 0.2522563875997467,
|
|
"rewards/margins": 0.1583661334906962,
|
|
"rewards/rejected": 0.09389025410905052,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 20.603500366210938,
|
|
"kl": 5.978942394256592,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36682372.453857794,
|
|
"logits/rejected": -37832872.73667205,
|
|
"logps/chosen": -502.98165658093797,
|
|
"logps/rejected": -383.53435480613894,
|
|
"loss": 0.5794,
|
|
"loss/base_kto": 3.853320837020874,
|
|
"metrics/advantage_var": 174.62965393066406,
|
|
"regularization/mmd": 0.7816373109817505,
|
|
"regularization/rkhs_norm": 150.60447692871094,
|
|
"rewards/chosen": 0.09290631131216906,
|
|
"rewards/margins": 0.13982160289653522,
|
|
"rewards/rejected": -0.046915291584366164,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 21.397480010986328,
|
|
"kl": 12.552741050720215,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35619206.0952381,
|
|
"logits/rejected": -37989193.25538462,
|
|
"logps/chosen": -494.6147817460317,
|
|
"logps/rejected": -360.1650480769231,
|
|
"loss": 0.5866,
|
|
"loss/base_kto": 3.8501617908477783,
|
|
"metrics/advantage_var": 204.6127166748047,
|
|
"regularization/mmd": 0.8428762555122375,
|
|
"regularization/rkhs_norm": 162.40391540527344,
|
|
"rewards/chosen": 0.21830361986917163,
|
|
"rewards/margins": 0.13981841854705623,
|
|
"rewards/rejected": 0.07848520132211538,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 15.790153503417969,
|
|
"kl": 15.475531578063965,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35700546.427921094,
|
|
"logits/rejected": -37738123.336553946,
|
|
"logps/chosen": -481.3101764036419,
|
|
"logps/rejected": -352.2597876409018,
|
|
"loss": 0.579,
|
|
"loss/base_kto": 3.777491807937622,
|
|
"metrics/advantage_var": 209.87998962402344,
|
|
"regularization/mmd": 0.8547720313072205,
|
|
"regularization/rkhs_norm": 164.69595336914062,
|
|
"rewards/chosen": 0.2858772104173582,
|
|
"rewards/margins": 0.19888717339919193,
|
|
"rewards/rejected": 0.08699003701816627,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 39.783966064453125,
|
|
"kl": 11.525238037109375,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35765464.42902208,
|
|
"logits/rejected": -37715473.436532505,
|
|
"logps/chosen": -489.69104889589903,
|
|
"logps/rejected": -387.1947078173375,
|
|
"loss": 0.5811,
|
|
"loss/base_kto": 3.787822723388672,
|
|
"metrics/advantage_var": 236.62136840820312,
|
|
"regularization/mmd": 0.8610712289810181,
|
|
"regularization/rkhs_norm": 165.90966796875,
|
|
"rewards/chosen": 0.1821763432740413,
|
|
"rewards/margins": 0.1777836617979089,
|
|
"rewards/rejected": 0.00439268147613242,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 16.868717193603516,
|
|
"kl": 3.994859457015991,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35069476.27018122,
|
|
"logits/rejected": -37011054.312035665,
|
|
"logps/chosen": -492.60337726523886,
|
|
"logps/rejected": -384.65583209509657,
|
|
"loss": 0.5817,
|
|
"loss/base_kto": 3.8289992809295654,
|
|
"metrics/advantage_var": 194.04469299316406,
|
|
"regularization/mmd": 0.8245702981948853,
|
|
"regularization/rkhs_norm": 158.87673950195312,
|
|
"rewards/chosen": -0.04185859139707964,
|
|
"rewards/margins": 0.14970385146748433,
|
|
"rewards/rejected": -0.19156244286456398,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 16.328460693359375,
|
|
"kl": 3.0741677284240723,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -37287643.428571425,
|
|
"logits/rejected": -38373464.22153846,
|
|
"logps/chosen": -507.6699404761905,
|
|
"logps/rejected": -377.856875,
|
|
"loss": 0.5845,
|
|
"loss/base_kto": 3.7946877479553223,
|
|
"metrics/advantage_var": 219.6715545654297,
|
|
"regularization/mmd": 0.8813549280166626,
|
|
"regularization/rkhs_norm": 169.8179168701172,
|
|
"rewards/chosen": -0.04087443276057168,
|
|
"rewards/margins": 0.19756701104952448,
|
|
"rewards/rejected": -0.23844144381009616,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 18.04525375366211,
|
|
"kl": 5.2681803703308105,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37049502.315789476,
|
|
"logits/rejected": -38178852.571428575,
|
|
"logps/chosen": -487.16395970394734,
|
|
"logps/rejected": -392.4440104166667,
|
|
"loss": 0.5801,
|
|
"loss/base_kto": 3.777139663696289,
|
|
"metrics/advantage_var": 222.96934509277344,
|
|
"regularization/mmd": 0.8638244867324829,
|
|
"regularization/rkhs_norm": 166.44015502929688,
|
|
"rewards/chosen": -0.014353609398791664,
|
|
"rewards/margins": 0.18498335080338002,
|
|
"rewards/rejected": -0.19933696020217168,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 18.46519660949707,
|
|
"kl": 4.6849565505981445,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -34838346.531645566,
|
|
"logits/rejected": -36299061.72839506,
|
|
"logps/chosen": -496.81788963607596,
|
|
"logps/rejected": -388.7009066358025,
|
|
"loss": 0.5854,
|
|
"loss/base_kto": 3.8158562183380127,
|
|
"metrics/advantage_var": 220.43008422851562,
|
|
"regularization/mmd": 0.866993248462677,
|
|
"regularization/rkhs_norm": 167.0507354736328,
|
|
"rewards/chosen": -0.006396480753451963,
|
|
"rewards/margins": 0.17232688983393232,
|
|
"rewards/rejected": -0.17872337058738427,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 17.094303131103516,
|
|
"kl": 15.332864761352539,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36305746.33918129,
|
|
"logits/rejected": -37991987.54362416,
|
|
"logps/chosen": -503.6694078947368,
|
|
"logps/rejected": -353.52385696308727,
|
|
"loss": 0.5782,
|
|
"loss/base_kto": 3.7261035442352295,
|
|
"metrics/advantage_var": 227.9002685546875,
|
|
"regularization/mmd": 0.8994961977005005,
|
|
"regularization/rkhs_norm": 173.31333923339844,
|
|
"rewards/chosen": 0.2943106088024831,
|
|
"rewards/margins": 0.23246875288928248,
|
|
"rewards/rejected": 0.061841855913200634,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.33824920654297,
|
|
"kl": 14.3850679397583,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35676741.530864194,
|
|
"logits/rejected": -37013601.215189874,
|
|
"logps/chosen": -485.3454861111111,
|
|
"logps/rejected": -350.10759493670884,
|
|
"loss": 0.5878,
|
|
"loss/base_kto": 3.7968785762786865,
|
|
"metrics/advantage_var": 234.15122985839844,
|
|
"regularization/mmd": 0.9053724408149719,
|
|
"regularization/rkhs_norm": 174.44554138183594,
|
|
"rewards/chosen": 0.25135205115801024,
|
|
"rewards/margins": 0.17415049762013446,
|
|
"rewards/rejected": 0.07720155353787579,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 18.95519256591797,
|
|
"kl": 7.190995693206787,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -35703291.82381729,
|
|
"logits/rejected": -38211138.782608695,
|
|
"logps/chosen": -489.71044045676996,
|
|
"logps/rejected": -365.9665011244378,
|
|
"loss": 0.5815,
|
|
"loss/base_kto": 3.773439407348633,
|
|
"metrics/advantage_var": 221.27774047851562,
|
|
"regularization/mmd": 0.8783413171768188,
|
|
"regularization/rkhs_norm": 169.2372283935547,
|
|
"rewards/chosen": 0.05311624299838142,
|
|
"rewards/margins": 0.18281556112496267,
|
|
"rewards/rejected": -0.12969931812658125,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 13.247111320495605,
|
|
"kl": 9.16240406036377,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -34967635.2,
|
|
"logits/rejected": -36332473.6,
|
|
"logps/chosen": -496.133203125,
|
|
"logps/rejected": -380.3676025390625,
|
|
"loss": 0.5856,
|
|
"loss/base_kto": 3.7572200298309326,
|
|
"metrics/advantage_var": 239.2593231201172,
|
|
"regularization/mmd": 0.9278042912483215,
|
|
"regularization/rkhs_norm": 178.76768493652344,
|
|
"rewards/chosen": 0.1451313853263855,
|
|
"rewards/margins": 0.22013087272644044,
|
|
"rewards/rejected": -0.07499948740005494,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 16.3098087310791,
|
|
"kl": 6.792578220367432,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36317901.88770865,
|
|
"logits/rejected": -36482547.632850245,
|
|
"logps/chosen": -458.5691388467375,
|
|
"logps/rejected": -377.06154388083735,
|
|
"loss": 0.5841,
|
|
"loss/base_kto": 3.8088531494140625,
|
|
"metrics/advantage_var": 216.6083526611328,
|
|
"regularization/mmd": 0.8642234802246094,
|
|
"regularization/rkhs_norm": 166.51707458496094,
|
|
"rewards/chosen": 0.04237878593942645,
|
|
"rewards/margins": 0.1907803135284965,
|
|
"rewards/rejected": -0.14840152758907005,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 11.708365440368652,
|
|
"kl": 11.88242244720459,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36764419.98753894,
|
|
"logits/rejected": -37439157.36677116,
|
|
"logps/chosen": -474.22605140186914,
|
|
"logps/rejected": -384.5523608934169,
|
|
"loss": 0.5828,
|
|
"loss/base_kto": 3.7636170387268066,
|
|
"metrics/advantage_var": 235.3195343017578,
|
|
"regularization/mmd": 0.8990959525108337,
|
|
"regularization/rkhs_norm": 173.23623657226562,
|
|
"rewards/chosen": 0.19089648285387462,
|
|
"rewards/margins": 0.21902219198098388,
|
|
"rewards/rejected": -0.028125709127109252,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 14.318680763244629,
|
|
"kl": 8.728950500488281,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -37032318.20839813,
|
|
"logits/rejected": -37374051.667189956,
|
|
"logps/chosen": -493.59958203732504,
|
|
"logps/rejected": -381.1712127158556,
|
|
"loss": 0.5837,
|
|
"loss/base_kto": 3.7761952877044678,
|
|
"metrics/advantage_var": 229.3874053955078,
|
|
"regularization/mmd": 0.8933205008506775,
|
|
"regularization/rkhs_norm": 172.1234130859375,
|
|
"rewards/chosen": 0.1257154070200112,
|
|
"rewards/margins": 0.2105761084088149,
|
|
"rewards/rejected": -0.08486070138880372,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 16.535734176635742,
|
|
"kl": 11.216611862182617,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36508873.6,
|
|
"logits/rejected": -37794348.8,
|
|
"logps/chosen": -500.35439453125,
|
|
"logps/rejected": -369.51689453125,
|
|
"loss": 0.5748,
|
|
"loss/base_kto": 3.7446205615997314,
|
|
"metrics/advantage_var": 202.92088317871094,
|
|
"regularization/mmd": 0.8536295294761658,
|
|
"regularization/rkhs_norm": 164.47581481933594,
|
|
"rewards/chosen": 0.18334336280822755,
|
|
"rewards/margins": 0.22333242297172548,
|
|
"rewards/rejected": -0.03998906016349792,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 16.610736846923828,
|
|
"kl": 9.801573753356934,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35763556.67415731,
|
|
"logits/rejected": -37100420.87062405,
|
|
"logps/chosen": -495.1868980738363,
|
|
"logps/rejected": -375.4642551369863,
|
|
"loss": 0.5756,
|
|
"loss/base_kto": 3.720391035079956,
|
|
"metrics/advantage_var": 217.6693115234375,
|
|
"regularization/mmd": 0.884712815284729,
|
|
"regularization/rkhs_norm": 170.4649200439453,
|
|
"rewards/chosen": 0.12955010721809793,
|
|
"rewards/margins": 0.2572847167900455,
|
|
"rewards/rejected": -0.12773460957194754,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 14.878124237060547,
|
|
"kl": 10.152867317199707,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35227657.95786062,
|
|
"logits/rejected": -35691506.09954751,
|
|
"logps/chosen": -483.7574959481361,
|
|
"logps/rejected": -374.82847850678735,
|
|
"loss": 0.5803,
|
|
"loss/base_kto": 3.7252635955810547,
|
|
"metrics/advantage_var": 236.1919403076172,
|
|
"regularization/mmd": 0.9174157381057739,
|
|
"regularization/rkhs_norm": 176.7660369873047,
|
|
"rewards/chosen": 0.06529909749085165,
|
|
"rewards/margins": 0.23877645446053253,
|
|
"rewards/rejected": -0.1734773569696809,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 17.19070053100586,
|
|
"kl": 4.2611212730407715,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35364382.68769716,
|
|
"logits/rejected": -36261510.7368421,
|
|
"logps/chosen": -497.1975059148265,
|
|
"logps/rejected": -363.3968895123839,
|
|
"loss": 0.5845,
|
|
"loss/base_kto": 3.769343137741089,
|
|
"metrics/advantage_var": 237.5447235107422,
|
|
"regularization/mmd": 0.9070194363594055,
|
|
"regularization/rkhs_norm": 174.7628936767578,
|
|
"rewards/chosen": -0.05222674950440223,
|
|
"rewards/margins": 0.21744712057045748,
|
|
"rewards/rejected": -0.2696738700748597,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 16.37181854248047,
|
|
"kl": 14.163302421569824,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35703692.33222591,
|
|
"logits/rejected": -36923225.864306785,
|
|
"logps/chosen": -476.10356104651163,
|
|
"logps/rejected": -371.15532817109147,
|
|
"loss": 0.5842,
|
|
"loss/base_kto": 3.721531629562378,
|
|
"metrics/advantage_var": 264.5733642578125,
|
|
"regularization/mmd": 0.9521066546440125,
|
|
"regularization/rkhs_norm": 183.45022583007812,
|
|
"rewards/chosen": 0.20643810893214026,
|
|
"rewards/margins": 0.2370058295085138,
|
|
"rewards/rejected": -0.030567720576373525,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 17.20627212524414,
|
|
"kl": 5.4861884117126465,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -34640203.43206107,
|
|
"logits/rejected": -34950722.3552,
|
|
"logps/chosen": -456.92643129770994,
|
|
"logps/rejected": -369.90875,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.8004844188690186,
|
|
"metrics/advantage_var": 219.798828125,
|
|
"regularization/mmd": 0.8696132898330688,
|
|
"regularization/rkhs_norm": 167.55552673339844,
|
|
"rewards/chosen": 0.05286537927525644,
|
|
"rewards/margins": 0.19684445642369394,
|
|
"rewards/rejected": -0.1439790771484375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 14.005735397338867,
|
|
"kl": 18.02949333190918,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35412104.53333333,
|
|
"logits/rejected": -34605636.53543307,
|
|
"logps/chosen": -474.1112403100775,
|
|
"logps/rejected": -372.8711614173228,
|
|
"loss": 0.5748,
|
|
"loss/base_kto": 3.701120138168335,
|
|
"metrics/advantage_var": 223.4566192626953,
|
|
"regularization/mmd": 0.8973054885864258,
|
|
"regularization/rkhs_norm": 172.89122009277344,
|
|
"rewards/chosen": 0.3634170413941376,
|
|
"rewards/margins": 0.25110125222398505,
|
|
"rewards/rejected": 0.11231578917015256,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 12.172685623168945,
|
|
"kl": 9.777069091796875,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -34705865.63549921,
|
|
"logits/rejected": -35008127.01386749,
|
|
"logps/chosen": -450.7083003169572,
|
|
"logps/rejected": -334.51926040061636,
|
|
"loss": 0.571,
|
|
"loss/base_kto": 3.7229297161102295,
|
|
"metrics/advantage_var": 203.33544921875,
|
|
"regularization/mmd": 0.8448325991630554,
|
|
"regularization/rkhs_norm": 162.7808380126953,
|
|
"rewards/chosen": 0.11691013342227119,
|
|
"rewards/margins": 0.26104623015913797,
|
|
"rewards/rejected": -0.1441360967368668,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 17.458974838256836,
|
|
"kl": 7.39281702041626,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -34642952.16586922,
|
|
"logits/rejected": -35806457.33537519,
|
|
"logps/chosen": -468.87250797448166,
|
|
"logps/rejected": -387.4709753062787,
|
|
"loss": 0.585,
|
|
"loss/base_kto": 3.754244327545166,
|
|
"metrics/advantage_var": 255.39453125,
|
|
"regularization/mmd": 0.9255840182304382,
|
|
"regularization/rkhs_norm": 178.33990478515625,
|
|
"rewards/chosen": -0.005259002795059716,
|
|
"rewards/margins": 0.23410685969058345,
|
|
"rewards/rejected": -0.23936586248564318,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 18.057920455932617,
|
|
"kl": 8.799531936645508,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -34666563.957767725,
|
|
"logits/rejected": -35867158.405186385,
|
|
"logps/chosen": -494.5712669683258,
|
|
"logps/rejected": -373.538239465154,
|
|
"loss": 0.5895,
|
|
"loss/base_kto": 3.738917589187622,
|
|
"metrics/advantage_var": 280.51007080078125,
|
|
"regularization/mmd": 0.977146565914154,
|
|
"regularization/rkhs_norm": 188.27488708496094,
|
|
"rewards/chosen": 0.09878811613287142,
|
|
"rewards/margins": 0.237137998533854,
|
|
"rewards/rejected": -0.1383498824009826,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 15.648366928100586,
|
|
"kl": 17.0057430267334,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37764959.37254902,
|
|
"logits/rejected": -37616206.83306321,
|
|
"logps/chosen": -507.4282145550528,
|
|
"logps/rejected": -382.2111780794165,
|
|
"loss": 0.5864,
|
|
"loss/base_kto": 3.7137954235076904,
|
|
"metrics/advantage_var": 272.029296875,
|
|
"regularization/mmd": 0.9773384928703308,
|
|
"regularization/rkhs_norm": 188.3118438720703,
|
|
"rewards/chosen": 0.3289657753876791,
|
|
"rewards/margins": 0.25797809870227784,
|
|
"rewards/rejected": 0.07098767668540126,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 13.922004699707031,
|
|
"kl": 11.66692066192627,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -35887049.25786164,
|
|
"logits/rejected": -37288495.701863356,
|
|
"logps/chosen": -494.5318887578616,
|
|
"logps/rejected": -356.20792895962734,
|
|
"loss": 0.5759,
|
|
"loss/base_kto": 3.684947967529297,
|
|
"metrics/advantage_var": 247.2151641845703,
|
|
"regularization/mmd": 0.9218670725822449,
|
|
"regularization/rkhs_norm": 177.62371826171875,
|
|
"rewards/chosen": 0.21588211539406446,
|
|
"rewards/margins": 0.2977201545316263,
|
|
"rewards/rejected": -0.08183803913756187,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 12.778936386108398,
|
|
"kl": 10.353636741638184,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35359593.025641024,
|
|
"logits/rejected": -36776095.219512194,
|
|
"logps/chosen": -466.09204727564105,
|
|
"logps/rejected": -383.8243378429878,
|
|
"loss": 0.5803,
|
|
"loss/base_kto": 3.743032932281494,
|
|
"metrics/advantage_var": 236.04234313964844,
|
|
"regularization/mmd": 0.8997423052787781,
|
|
"regularization/rkhs_norm": 173.3607635498047,
|
|
"rewards/chosen": 0.08536517314421825,
|
|
"rewards/margins": 0.2294842497567373,
|
|
"rewards/rejected": -0.14411907661251905,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 12.796786308288574,
|
|
"kl": 10.059364318847656,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37100506.91023622,
|
|
"logits/rejected": -38145832.086821705,
|
|
"logps/chosen": -483.1859744094488,
|
|
"logps/rejected": -385.253003875969,
|
|
"loss": 0.5757,
|
|
"loss/base_kto": 3.691363573074341,
|
|
"metrics/advantage_var": 237.73448181152344,
|
|
"regularization/mmd": 0.9140893816947937,
|
|
"regularization/rkhs_norm": 176.12513732910156,
|
|
"rewards/chosen": 0.1902135894054503,
|
|
"rewards/margins": 0.2664576242834935,
|
|
"rewards/rejected": -0.07624403487804324,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 18.834814071655273,
|
|
"kl": 11.848525047302246,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -38043121.28290469,
|
|
"logits/rejected": -38057669.686591275,
|
|
"logps/chosen": -480.3556637670197,
|
|
"logps/rejected": -382.6019789983845,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.799571990966797,
|
|
"metrics/advantage_var": 250.9722137451172,
|
|
"regularization/mmd": 0.9297453165054321,
|
|
"regularization/rkhs_norm": 179.14169311523438,
|
|
"rewards/chosen": 0.17881142389756288,
|
|
"rewards/margins": 0.1670404575325683,
|
|
"rewards/rejected": 0.011770966364994574,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 18.071857452392578,
|
|
"kl": 10.782426834106445,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36713761.42809917,
|
|
"logits/rejected": -37365374.672592595,
|
|
"logps/chosen": -463.689979338843,
|
|
"logps/rejected": -371.9513425925926,
|
|
"loss": 0.5846,
|
|
"loss/base_kto": 3.789031982421875,
|
|
"metrics/advantage_var": 230.16294860839844,
|
|
"regularization/mmd": 0.8880990147590637,
|
|
"regularization/rkhs_norm": 171.11734008789062,
|
|
"rewards/chosen": 0.1387394802629455,
|
|
"rewards/margins": 0.16395804263635405,
|
|
"rewards/rejected": -0.025218562373408564,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 13.347097396850586,
|
|
"kl": 8.719230651855469,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36403756.82949309,
|
|
"logits/rejected": -37686954.12400636,
|
|
"logps/chosen": -487.795410906298,
|
|
"logps/rejected": -384.7078696343402,
|
|
"loss": 0.578,
|
|
"loss/base_kto": 3.6972596645355225,
|
|
"metrics/advantage_var": 266.856689453125,
|
|
"regularization/mmd": 0.9265327453613281,
|
|
"regularization/rkhs_norm": 178.522705078125,
|
|
"rewards/chosen": 0.11646345756753432,
|
|
"rewards/margins": 0.28006160762979215,
|
|
"rewards/rejected": -0.1635981500622578,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 21.081863403320312,
|
|
"kl": 10.20801067352295,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36710791.90123457,
|
|
"logits/rejected": -37301760.0,
|
|
"logps/chosen": -505.49508101851853,
|
|
"logps/rejected": -377.8860512262658,
|
|
"loss": 0.5778,
|
|
"loss/base_kto": 3.713108777999878,
|
|
"metrics/advantage_var": 232.65188598632812,
|
|
"regularization/mmd": 0.9090795516967773,
|
|
"regularization/rkhs_norm": 175.1598358154297,
|
|
"rewards/chosen": 0.13450986367684822,
|
|
"rewards/margins": 0.23079654078387452,
|
|
"rewards/rejected": -0.0962866771070263,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.837052345275879,
|
|
"kl": 14.241604804992676,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35378290.14012739,
|
|
"logits/rejected": -37924980.220858894,
|
|
"logps/chosen": -472.6911325636943,
|
|
"logps/rejected": -360.00575153374234,
|
|
"loss": 0.5734,
|
|
"loss/base_kto": 3.7171380519866943,
|
|
"metrics/advantage_var": 216.7089385986328,
|
|
"regularization/mmd": 0.8703489303588867,
|
|
"regularization/rkhs_norm": 167.69729614257812,
|
|
"rewards/chosen": 0.19872902305262863,
|
|
"rewards/margins": 0.24355476709645862,
|
|
"rewards/rejected": -0.04482574404382998,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 16.90869140625,
|
|
"kl": 15.4988431930542,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36117483.10204082,
|
|
"logits/rejected": -36762427.47474747,
|
|
"logps/chosen": -481.2785167638484,
|
|
"logps/rejected": -401.38862584175087,
|
|
"loss": 0.5882,
|
|
"loss/base_kto": 3.7433693408966064,
|
|
"metrics/advantage_var": 260.5050354003906,
|
|
"regularization/mmd": 0.962075412273407,
|
|
"regularization/rkhs_norm": 185.37098693847656,
|
|
"rewards/chosen": 0.23253413609095983,
|
|
"rewards/margins": 0.20535857956130785,
|
|
"rewards/rejected": 0.02717555652965199,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 16.020484924316406,
|
|
"kl": 14.999961853027344,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36411713.37846154,
|
|
"logits/rejected": -37450761.78343949,
|
|
"logps/chosen": -479.30384615384617,
|
|
"logps/rejected": -370.94436703821657,
|
|
"loss": 0.5736,
|
|
"loss/base_kto": 3.504671812057495,
|
|
"metrics/advantage_var": 369.0525817871094,
|
|
"regularization/mmd": 1.0838483572006226,
|
|
"regularization/rkhs_norm": 208.833984375,
|
|
"rewards/chosen": 0.3749387066180889,
|
|
"rewards/margins": 0.5237920815123465,
|
|
"rewards/rejected": -0.14885337489425757,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 13.673979759216309,
|
|
"kl": 9.484530448913574,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35978683.51592357,
|
|
"logits/rejected": -36763145.42331288,
|
|
"logps/chosen": -466.19342157643314,
|
|
"logps/rejected": -360.1636790644172,
|
|
"loss": 0.5727,
|
|
"loss/base_kto": 3.2092158794403076,
|
|
"metrics/advantage_var": 611.8053588867188,
|
|
"regularization/mmd": 1.3727378845214844,
|
|
"regularization/rkhs_norm": 264.4967041015625,
|
|
"rewards/chosen": 0.4779043015401075,
|
|
"rewards/margins": 0.9047891457993579,
|
|
"rewards/rejected": -0.4268848442592504,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 11.30276870727539,
|
|
"kl": 14.168227195739746,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36663465.04126984,
|
|
"logits/rejected": -38344281.79692308,
|
|
"logps/chosen": -479.315376984127,
|
|
"logps/rejected": -402.0303125,
|
|
"loss": 0.5832,
|
|
"loss/base_kto": 3.178797960281372,
|
|
"metrics/advantage_var": 1516.0076904296875,
|
|
"regularization/mmd": 1.486470103263855,
|
|
"regularization/rkhs_norm": 286.41046142578125,
|
|
"rewards/chosen": 0.5310749356708829,
|
|
"rewards/margins": 0.9633089162898734,
|
|
"rewards/rejected": -0.4322339806189904,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 10.916857719421387,
|
|
"kl": 9.80307388305664,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36472250.03726708,
|
|
"logits/rejected": -37288348.17610063,
|
|
"logps/chosen": -484.9952445652174,
|
|
"logps/rejected": -401.23388364779873,
|
|
"loss": 0.5754,
|
|
"loss/base_kto": 3.1596202850341797,
|
|
"metrics/advantage_var": 647.99560546875,
|
|
"regularization/mmd": 1.4439016580581665,
|
|
"regularization/rkhs_norm": 278.20843505859375,
|
|
"rewards/chosen": 0.4981846779769992,
|
|
"rewards/margins": 0.9537182634354309,
|
|
"rewards/rejected": -0.4555335854584316,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 16.762218475341797,
|
|
"kl": 9.19613265991211,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -37068884.3866878,
|
|
"logits/rejected": -37087137.33127889,
|
|
"logps/chosen": -506.1333696513471,
|
|
"logps/rejected": -366.7082530816641,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.1624512672424316,
|
|
"metrics/advantage_var": 606.7392578125,
|
|
"regularization/mmd": 1.3861886262893677,
|
|
"regularization/rkhs_norm": 267.0883483886719,
|
|
"rewards/chosen": 0.4608260697305864,
|
|
"rewards/margins": 0.9256741928195935,
|
|
"rewards/rejected": -0.46484812308900714,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 14.93901252746582,
|
|
"kl": 14.44323444366455,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34925424.41910632,
|
|
"logits/rejected": -36761541.57844691,
|
|
"logps/chosen": -485.94655238828966,
|
|
"logps/rejected": -382.1991135103011,
|
|
"loss": 0.5686,
|
|
"loss/base_kto": 3.1689491271972656,
|
|
"metrics/advantage_var": 631.2672729492188,
|
|
"regularization/mmd": 1.3795952796936035,
|
|
"regularization/rkhs_norm": 265.8179931640625,
|
|
"rewards/chosen": 0.5743349425046947,
|
|
"rewards/margins": 0.9278340512559948,
|
|
"rewards/rejected": -0.35349910875130003,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 14.874568939208984,
|
|
"kl": 12.614941596984863,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36720434.8844376,
|
|
"logits/rejected": -37408209.7496038,
|
|
"logps/chosen": -480.2295839753467,
|
|
"logps/rejected": -381.60831022187006,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.21881365776062,
|
|
"metrics/advantage_var": 558.1520385742188,
|
|
"regularization/mmd": 1.319035291671753,
|
|
"regularization/rkhs_norm": 254.1493682861328,
|
|
"rewards/chosen": 0.5033026226496292,
|
|
"rewards/margins": 0.84659389170111,
|
|
"rewards/rejected": -0.3432912690514808,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 49.197689056396484,
|
|
"kl": 17.997711181640625,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35001093.972006224,
|
|
"logits/rejected": -37126870.605965465,
|
|
"logps/chosen": -467.9345839813375,
|
|
"logps/rejected": -369.26157770800626,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.2193355560302734,
|
|
"metrics/advantage_var": 537.2255859375,
|
|
"regularization/mmd": 1.295191764831543,
|
|
"regularization/rkhs_norm": 249.55528259277344,
|
|
"rewards/chosen": 0.5455483633845256,
|
|
"rewards/margins": 0.8122700080031604,
|
|
"rewards/rejected": -0.26672164461863473,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 15.033616065979004,
|
|
"kl": 16.166486740112305,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36191353.49152543,
|
|
"logits/rejected": -37762445.5911252,
|
|
"logps/chosen": -514.6885111710324,
|
|
"logps/rejected": -379.8275059429477,
|
|
"loss": 0.5708,
|
|
"loss/base_kto": 3.178814172744751,
|
|
"metrics/advantage_var": 581.7435913085938,
|
|
"regularization/mmd": 1.3872652053833008,
|
|
"regularization/rkhs_norm": 267.29583740234375,
|
|
"rewards/chosen": 0.579269150556144,
|
|
"rewards/margins": 0.9180882160430943,
|
|
"rewards/rejected": -0.3388190654869503,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 13.95463752746582,
|
|
"kl": 16.31585121154785,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35213970.62174579,
|
|
"logits/rejected": -36111237.51196172,
|
|
"logps/chosen": -506.10681470137825,
|
|
"logps/rejected": -374.78401614832535,
|
|
"loss": 0.5613,
|
|
"loss/base_kto": 3.1615407466888428,
|
|
"metrics/advantage_var": 577.4911499023438,
|
|
"regularization/mmd": 1.3287709951400757,
|
|
"regularization/rkhs_norm": 256.0252380371094,
|
|
"rewards/chosen": 0.5943864293704537,
|
|
"rewards/margins": 0.9013198917050456,
|
|
"rewards/rejected": -0.3069334623345918,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 14.795431137084961,
|
|
"kl": 15.70853328704834,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35944115.2,
|
|
"logits/rejected": -37309516.8,
|
|
"logps/chosen": -460.71923828125,
|
|
"logps/rejected": -373.9236328125,
|
|
"loss": 0.5632,
|
|
"loss/base_kto": 3.1640241146087646,
|
|
"metrics/advantage_var": 581.5676879882812,
|
|
"regularization/mmd": 1.34121572971344,
|
|
"regularization/rkhs_norm": 258.4230651855469,
|
|
"rewards/chosen": 0.5741420269012452,
|
|
"rewards/margins": 0.8985859870910645,
|
|
"rewards/rejected": -0.32444396018981936,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 11.119890213012695,
|
|
"kl": 15.3007230758667,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -36073831.064935066,
|
|
"logits/rejected": -37801724.915662654,
|
|
"logps/chosen": -469.5836546266234,
|
|
"logps/rejected": -368.4045557228916,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.218740940093994,
|
|
"metrics/advantage_var": 471.51123046875,
|
|
"regularization/mmd": 1.2567896842956543,
|
|
"regularization/rkhs_norm": 242.156005859375,
|
|
"rewards/chosen": 0.47081776408406045,
|
|
"rewards/margins": 0.7942033474330815,
|
|
"rewards/rejected": -0.32338558334902107,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 14.737159729003906,
|
|
"kl": 14.101458549499512,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34087149.06752411,
|
|
"logits/rejected": -36154109.6656535,
|
|
"logps/chosen": -475.9912580385852,
|
|
"logps/rejected": -368.0076462765957,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.1185109615325928,
|
|
"metrics/advantage_var": 601.0817260742188,
|
|
"regularization/mmd": 1.3650661706924438,
|
|
"regularization/rkhs_norm": 263.0185546875,
|
|
"rewards/chosen": 0.5136829633804763,
|
|
"rewards/margins": 0.9522213031054952,
|
|
"rewards/rejected": -0.438538339725019,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 13.771965980529785,
|
|
"kl": 16.108701705932617,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -36075593.25660964,
|
|
"logits/rejected": -36407508.19466248,
|
|
"logps/chosen": -491.42029548989115,
|
|
"logps/rejected": -398.88383045525904,
|
|
"loss": 0.5698,
|
|
"loss/base_kto": 3.122931718826294,
|
|
"metrics/advantage_var": 642.3717651367188,
|
|
"regularization/mmd": 1.435685396194458,
|
|
"regularization/rkhs_norm": 276.6253356933594,
|
|
"rewards/chosen": 0.5806941719114502,
|
|
"rewards/margins": 0.9448497720834601,
|
|
"rewards/rejected": -0.36415560017200993,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 11.923416137695312,
|
|
"kl": 12.88659954071045,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34624768.79256966,
|
|
"logits/rejected": -36532085.0977918,
|
|
"logps/chosen": -486.4784249226006,
|
|
"logps/rejected": -378.5344785094637,
|
|
"loss": 0.5644,
|
|
"loss/base_kto": 3.1559431552886963,
|
|
"metrics/advantage_var": 554.8573608398438,
|
|
"regularization/mmd": 1.3591049909591675,
|
|
"regularization/rkhs_norm": 261.86993408203125,
|
|
"rewards/chosen": 0.5736723625253967,
|
|
"rewards/margins": 0.9181868904329036,
|
|
"rewards/rejected": -0.3445145279075069,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 11.900761604309082,
|
|
"kl": 14.61228084564209,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35385788.10608424,
|
|
"logits/rejected": -35233836.87010954,
|
|
"logps/chosen": -473.9101989079563,
|
|
"logps/rejected": -377.85785895931144,
|
|
"loss": 0.5651,
|
|
"loss/base_kto": 3.23760986328125,
|
|
"metrics/advantage_var": 529.1963500976562,
|
|
"regularization/mmd": 1.2834538221359253,
|
|
"regularization/rkhs_norm": 247.2936553955078,
|
|
"rewards/chosen": 0.49898316000999415,
|
|
"rewards/margins": 0.8162707185044068,
|
|
"rewards/rejected": -0.31728755849441265,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 11.39002513885498,
|
|
"kl": 11.355319023132324,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -33878128.90658499,
|
|
"logits/rejected": -36059887.2599681,
|
|
"logps/chosen": -459.093989280245,
|
|
"logps/rejected": -377.0118620414673,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.210296630859375,
|
|
"metrics/advantage_var": 507.0902404785156,
|
|
"regularization/mmd": 1.2828924655914307,
|
|
"regularization/rkhs_norm": 247.18545532226562,
|
|
"rewards/chosen": 0.4646820021624952,
|
|
"rewards/margins": 0.8663505203262463,
|
|
"rewards/rejected": -0.401668518163751,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 12.310845375061035,
|
|
"kl": 19.80482292175293,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35533879.26349206,
|
|
"logits/rejected": -36472617.74769231,
|
|
"logps/chosen": -470.63665674603175,
|
|
"logps/rejected": -366.924375,
|
|
"loss": 0.5716,
|
|
"loss/base_kto": 3.2418675422668457,
|
|
"metrics/advantage_var": 535.7623901367188,
|
|
"regularization/mmd": 1.3306423425674438,
|
|
"regularization/rkhs_norm": 256.38580322265625,
|
|
"rewards/chosen": 0.5851631285652282,
|
|
"rewards/margins": 0.7435311658624038,
|
|
"rewards/rejected": -0.15836803729717547,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 9.959711074829102,
|
|
"kl": 12.635095596313477,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -33910771.45482389,
|
|
"logits/rejected": -34576929.480063796,
|
|
"logps/chosen": -470.28493491577336,
|
|
"logps/rejected": -374.89493620414675,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.207439422607422,
|
|
"metrics/advantage_var": 512.9755249023438,
|
|
"regularization/mmd": 1.2700233459472656,
|
|
"regularization/rkhs_norm": 244.7058563232422,
|
|
"rewards/chosen": 0.4744691717314318,
|
|
"rewards/margins": 0.8727347852447517,
|
|
"rewards/rejected": -0.39826561351331985,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 13.300355911254883,
|
|
"kl": 16.797433853149414,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35690637.68553459,
|
|
"logits/rejected": -36165212.22360248,
|
|
"logps/chosen": -485.21658805031444,
|
|
"logps/rejected": -377.45455648291926,
|
|
"loss": 0.5691,
|
|
"loss/base_kto": 3.1995484828948975,
|
|
"metrics/advantage_var": 560.8448486328125,
|
|
"regularization/mmd": 1.352962851524353,
|
|
"regularization/rkhs_norm": 260.6864929199219,
|
|
"rewards/chosen": 0.5934259666586822,
|
|
"rewards/margins": 0.8719599969806445,
|
|
"rewards/rejected": -0.27853403032196233,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 11.905561447143555,
|
|
"kl": 6.885768890380859,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35484328.464516126,
|
|
"logits/rejected": -36705503.41818182,
|
|
"logps/chosen": -499.04002016129033,
|
|
"logps/rejected": -385.2001420454545,
|
|
"loss": 0.5667,
|
|
"loss/base_kto": 3.2374038696289062,
|
|
"metrics/advantage_var": 552.7958374023438,
|
|
"regularization/mmd": 1.296007752418518,
|
|
"regularization/rkhs_norm": 249.7124786376953,
|
|
"rewards/chosen": 0.3102018540905368,
|
|
"rewards/margins": 0.8404505076179989,
|
|
"rewards/rejected": -0.5302486535274621,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 10.156408309936523,
|
|
"kl": 11.827449798583984,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34034260.936434105,
|
|
"logits/rejected": -36051140.73700787,
|
|
"logps/chosen": -480.66143410852715,
|
|
"logps/rejected": -356.6687992125984,
|
|
"loss": 0.5631,
|
|
"loss/base_kto": 3.159776449203491,
|
|
"metrics/advantage_var": 559.4918212890625,
|
|
"regularization/mmd": 1.3446849584579468,
|
|
"regularization/rkhs_norm": 259.0915222167969,
|
|
"rewards/chosen": 0.5264409028282461,
|
|
"rewards/margins": 0.9246205893670625,
|
|
"rewards/rejected": -0.3981796865388164,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 11.013405799865723,
|
|
"kl": 13.303488731384277,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34680534.913580246,
|
|
"logits/rejected": -36481876.25316456,
|
|
"logps/chosen": -499.7567515432099,
|
|
"logps/rejected": -371.2193433544304,
|
|
"loss": 0.5589,
|
|
"loss/base_kto": 3.1792984008789062,
|
|
"metrics/advantage_var": 509.5606384277344,
|
|
"regularization/mmd": 1.2922959327697754,
|
|
"regularization/rkhs_norm": 248.99729919433594,
|
|
"rewards/chosen": 0.5382480385862751,
|
|
"rewards/margins": 0.8853367678652259,
|
|
"rewards/rejected": -0.34708872927895074,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 12.741703987121582,
|
|
"kl": 11.428122520446777,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33385719.716486905,
|
|
"logits/rejected": -34775533.33755943,
|
|
"logps/chosen": -460.9606606317411,
|
|
"logps/rejected": -354.9784815768621,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.305621385574341,
|
|
"metrics/advantage_var": 438.2381896972656,
|
|
"regularization/mmd": 1.1939852237701416,
|
|
"regularization/rkhs_norm": 230.05496215820312,
|
|
"rewards/chosen": 0.483033825326223,
|
|
"rewards/margins": 0.7381768110616009,
|
|
"rewards/rejected": -0.25514298573537786,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 11.366232872009277,
|
|
"kl": 5.900028228759766,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35032320.0,
|
|
"logits/rejected": -36894805.333333336,
|
|
"logps/chosen": -504.27163856907896,
|
|
"logps/rejected": -380.8592587425595,
|
|
"loss": 0.5669,
|
|
"loss/base_kto": 3.2237019538879395,
|
|
"metrics/advantage_var": 545.4990234375,
|
|
"regularization/mmd": 1.3115177154541016,
|
|
"regularization/rkhs_norm": 252.7008819580078,
|
|
"rewards/chosen": 0.33717579590646846,
|
|
"rewards/margins": 0.8409306387554734,
|
|
"rewards/rejected": -0.5037548428490048,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 12.256927490234375,
|
|
"kl": 13.522027969360352,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -33599860.36363637,
|
|
"logits/rejected": -34542833.80031696,
|
|
"logps/chosen": -489.6143586286595,
|
|
"logps/rejected": -366.5445721077655,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.1496124267578125,
|
|
"metrics/advantage_var": 518.9310913085938,
|
|
"regularization/mmd": 1.2890504598617554,
|
|
"regularization/rkhs_norm": 248.3719482421875,
|
|
"rewards/chosen": 0.5277198356546129,
|
|
"rewards/margins": 0.8914266148323193,
|
|
"rewards/rejected": -0.3637067791777065,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 12.442972183227539,
|
|
"kl": 14.00787353515625,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33877180.34003091,
|
|
"logits/rejected": -34560682.666666664,
|
|
"logps/chosen": -470.5526468315301,
|
|
"logps/rejected": -361.9406101895735,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.203158140182495,
|
|
"metrics/advantage_var": 506.0343933105469,
|
|
"regularization/mmd": 1.2659547328948975,
|
|
"regularization/rkhs_norm": 243.921875,
|
|
"rewards/chosen": 0.5248600813852395,
|
|
"rewards/margins": 0.8424741621668598,
|
|
"rewards/rejected": -0.31761408078162023,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 13.786334037780762,
|
|
"kl": 15.736841201782227,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -35004769.82723279,
|
|
"logits/rejected": -34845584.50921273,
|
|
"logps/chosen": -471.00155563689606,
|
|
"logps/rejected": -361.7496335845896,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.2126832008361816,
|
|
"metrics/advantage_var": 481.12945556640625,
|
|
"regularization/mmd": 1.2475121021270752,
|
|
"regularization/rkhs_norm": 240.36843872070312,
|
|
"rewards/chosen": 0.6033123234180545,
|
|
"rewards/margins": 0.8328046898318291,
|
|
"rewards/rejected": -0.2294923664137746,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 11.664563179016113,
|
|
"kl": 20.466232299804688,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35302931.482496195,
|
|
"logits/rejected": -37887283.36436597,
|
|
"logps/chosen": -465.6245719178082,
|
|
"logps/rejected": -364.2195274879615,
|
|
"loss": 0.5596,
|
|
"loss/base_kto": 3.2070748805999756,
|
|
"metrics/advantage_var": 491.7352600097656,
|
|
"regularization/mmd": 1.269645094871521,
|
|
"regularization/rkhs_norm": 244.6329803466797,
|
|
"rewards/chosen": 0.6266446048266268,
|
|
"rewards/margins": 0.8284691175838561,
|
|
"rewards/rejected": -0.20182451275722937,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 12.147101402282715,
|
|
"kl": 20.791614532470703,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35249375.14029851,
|
|
"logits/rejected": -36237849.18032787,
|
|
"logps/chosen": -464.20289179104475,
|
|
"logps/rejected": -395.3352971311475,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.1634483337402344,
|
|
"metrics/advantage_var": 613.6397094726562,
|
|
"regularization/mmd": 1.3485749959945679,
|
|
"regularization/rkhs_norm": 259.841064453125,
|
|
"rewards/chosen": 0.7005729561421409,
|
|
"rewards/margins": 0.8940029593239736,
|
|
"rewards/rejected": -0.19343000318183273,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 12.986946105957031,
|
|
"kl": 17.93374252319336,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34984960.0,
|
|
"logits/rejected": -35616488.44171779,
|
|
"logps/chosen": -454.2381568471338,
|
|
"logps/rejected": -392.0549750766871,
|
|
"loss": 0.5702,
|
|
"loss/base_kto": 3.215028762817383,
|
|
"metrics/advantage_var": 649.096923828125,
|
|
"regularization/mmd": 1.3462086915969849,
|
|
"regularization/rkhs_norm": 259.3851013183594,
|
|
"rewards/chosen": 0.6024145624440187,
|
|
"rewards/margins": 0.8665838224807446,
|
|
"rewards/rejected": -0.2641692600367259,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 17.55329132080078,
|
|
"kl": 14.653973579406738,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33923944.81582953,
|
|
"logits/rejected": -34592531.3130016,
|
|
"logps/chosen": -499.34103881278537,
|
|
"logps/rejected": -377.31270064205455,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.2681057453155518,
|
|
"metrics/advantage_var": 524.8186645507812,
|
|
"regularization/mmd": 1.2558765411376953,
|
|
"regularization/rkhs_norm": 241.98008728027344,
|
|
"rewards/chosen": 0.5251274399198536,
|
|
"rewards/margins": 0.776356764383136,
|
|
"rewards/rejected": -0.2512293244632825,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 14.489201545715332,
|
|
"kl": 12.35971450805664,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -32918209.752012882,
|
|
"logits/rejected": -34491322.07587253,
|
|
"logps/chosen": -479.5342190016103,
|
|
"logps/rejected": -367.57833839150226,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.199578285217285,
|
|
"metrics/advantage_var": 461.0147399902344,
|
|
"regularization/mmd": 1.238900065422058,
|
|
"regularization/rkhs_norm": 238.7090606689453,
|
|
"rewards/chosen": 0.48035454250956117,
|
|
"rewards/margins": 0.8472090582219924,
|
|
"rewards/rejected": -0.3668545157124312,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 18.35354232788086,
|
|
"kl": 11.523287773132324,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34819714.97674418,
|
|
"logits/rejected": -36969760.65511811,
|
|
"logps/chosen": -503.99040697674417,
|
|
"logps/rejected": -388.50851377952756,
|
|
"loss": 0.5693,
|
|
"loss/base_kto": 3.199770450592041,
|
|
"metrics/advantage_var": 548.7518920898438,
|
|
"regularization/mmd": 1.3547416925430298,
|
|
"regularization/rkhs_norm": 261.02923583984375,
|
|
"rewards/chosen": 0.4939948503361192,
|
|
"rewards/margins": 0.8702494351817147,
|
|
"rewards/rejected": -0.3762545848455955,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 10.568586349487305,
|
|
"kl": 12.54379940032959,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -32372905.28363047,
|
|
"logits/rejected": -34948523.82503771,
|
|
"logps/chosen": -468.6657718800648,
|
|
"logps/rejected": -366.2825697586727,
|
|
"loss": 0.5583,
|
|
"loss/base_kto": 3.1970772743225098,
|
|
"metrics/advantage_var": 500.9302673339844,
|
|
"regularization/mmd": 1.2695538997650146,
|
|
"regularization/rkhs_norm": 244.61538696289062,
|
|
"rewards/chosen": 0.524188816064374,
|
|
"rewards/margins": 0.8467278104406701,
|
|
"rewards/rejected": -0.3225389943762962,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 13.829594612121582,
|
|
"kl": 7.175731658935547,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -33422147.32598425,
|
|
"logits/rejected": -35442037.08527132,
|
|
"logps/chosen": -455.57632874015746,
|
|
"logps/rejected": -364.9403343023256,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.2825868129730225,
|
|
"metrics/advantage_var": 501.10504150390625,
|
|
"regularization/mmd": 1.255201816558838,
|
|
"regularization/rkhs_norm": 241.8500518798828,
|
|
"rewards/chosen": 0.37917466050996557,
|
|
"rewards/margins": 0.7758713810768261,
|
|
"rewards/rejected": -0.39669672056686045,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 12.887574195861816,
|
|
"kl": 8.26279067993164,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35577927.066246055,
|
|
"logits/rejected": -36394488.0743034,
|
|
"logps/chosen": -510.4785094637224,
|
|
"logps/rejected": -375.7524671052632,
|
|
"loss": 0.5703,
|
|
"loss/base_kto": 3.154165029525757,
|
|
"metrics/advantage_var": 658.8799438476562,
|
|
"regularization/mmd": 1.4083423614501953,
|
|
"regularization/rkhs_norm": 271.3569030761719,
|
|
"rewards/chosen": 0.40889289928159506,
|
|
"rewards/margins": 0.9694168152367517,
|
|
"rewards/rejected": -0.5605239159551567,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 13.918904304504395,
|
|
"kl": 17.184967041015625,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35319389.23961661,
|
|
"logits/rejected": -36476683.74311927,
|
|
"logps/chosen": -493.04537739616615,
|
|
"logps/rejected": -365.0766915137615,
|
|
"loss": 0.562,
|
|
"loss/base_kto": 3.172517776489258,
|
|
"metrics/advantage_var": 538.2606811523438,
|
|
"regularization/mmd": 1.3230832815170288,
|
|
"regularization/rkhs_norm": 254.92935180664062,
|
|
"rewards/chosen": 0.5746032422343001,
|
|
"rewards/margins": 0.862835328135963,
|
|
"rewards/rejected": -0.2882320859016628,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 10.085333824157715,
|
|
"kl": 12.689724922180176,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35791337.66720517,
|
|
"logits/rejected": -35233637.083207265,
|
|
"logps/chosen": -485.1252524232633,
|
|
"logps/rejected": -404.0833018154312,
|
|
"loss": 0.5569,
|
|
"loss/base_kto": 3.140934705734253,
|
|
"metrics/advantage_var": 536.6046142578125,
|
|
"regularization/mmd": 1.3145030736923218,
|
|
"regularization/rkhs_norm": 253.276123046875,
|
|
"rewards/chosen": 0.4152452410326636,
|
|
"rewards/margins": 0.882829386444445,
|
|
"rewards/rejected": -0.4675841454117814,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 12.90168285369873,
|
|
"kl": 9.561026573181152,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35614466.702865764,
|
|
"logits/rejected": -35578731.811382115,
|
|
"logps/chosen": -473.2989253393665,
|
|
"logps/rejected": -404.78363821138214,
|
|
"loss": 0.5397,
|
|
"loss/base_kto": 3.083563804626465,
|
|
"metrics/advantage_var": 452.0966796875,
|
|
"regularization/mmd": 1.2337735891342163,
|
|
"regularization/rkhs_norm": 237.7213134765625,
|
|
"rewards/chosen": 0.5207481787035728,
|
|
"rewards/margins": 0.9779926051517436,
|
|
"rewards/rejected": -0.4572444264481707,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 11.978687286376953,
|
|
"kl": 10.999704360961914,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35162362.709062,
|
|
"logits/rejected": -35614141.14900154,
|
|
"logps/chosen": -474.64109697933225,
|
|
"logps/rejected": -371.28619431643625,
|
|
"loss": 0.5278,
|
|
"loss/base_kto": 3.097466230392456,
|
|
"metrics/advantage_var": 362.7237243652344,
|
|
"regularization/mmd": 1.124959945678711,
|
|
"regularization/rkhs_norm": 216.75527954101562,
|
|
"rewards/chosen": 0.5506757727109003,
|
|
"rewards/margins": 0.9320460241569928,
|
|
"rewards/rejected": -0.38137025144609255,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 8.585254669189453,
|
|
"kl": 8.297131538391113,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33853078.78317152,
|
|
"logits/rejected": -35635599.081571,
|
|
"logps/chosen": -481.5484425566343,
|
|
"logps/rejected": -367.30579682779455,
|
|
"loss": 0.5317,
|
|
"loss/base_kto": 3.094708204269409,
|
|
"metrics/advantage_var": 429.740966796875,
|
|
"regularization/mmd": 1.1589012145996094,
|
|
"regularization/rkhs_norm": 223.29502868652344,
|
|
"rewards/chosen": 0.49697243897274473,
|
|
"rewards/margins": 0.9603473346044715,
|
|
"rewards/rejected": -0.4633748956317268,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 14.268289566040039,
|
|
"kl": 9.894978523254395,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -33966447.58974359,
|
|
"logits/rejected": -35747574.63414634,
|
|
"logps/chosen": -469.7408854166667,
|
|
"logps/rejected": -369.0608565167683,
|
|
"loss": 0.5221,
|
|
"loss/base_kto": 3.08331561088562,
|
|
"metrics/advantage_var": 355.4896545410156,
|
|
"regularization/mmd": 1.0933945178985596,
|
|
"regularization/rkhs_norm": 210.6732940673828,
|
|
"rewards/chosen": 0.4529416010929988,
|
|
"rewards/margins": 0.9207873317582522,
|
|
"rewards/rejected": -0.4678457306652534,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 12.006796836853027,
|
|
"kl": 6.364843368530273,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34064046.9068323,
|
|
"logits/rejected": -34977965.88679245,
|
|
"logps/chosen": -511.1766304347826,
|
|
"logps/rejected": -377.6381682389937,
|
|
"loss": 0.5302,
|
|
"loss/base_kto": 3.134618043899536,
|
|
"metrics/advantage_var": 341.3658752441406,
|
|
"regularization/mmd": 1.106918215751648,
|
|
"regularization/rkhs_norm": 213.27903747558594,
|
|
"rewards/chosen": 0.4341693191054445,
|
|
"rewards/margins": 0.9212265071275062,
|
|
"rewards/rejected": -0.4870571880220617,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 14.187989234924316,
|
|
"kl": 13.8912992477417,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33409737.556561086,
|
|
"logits/rejected": -34950779.64343598,
|
|
"logps/chosen": -468.34931184012066,
|
|
"logps/rejected": -370.4262560777958,
|
|
"loss": 0.5263,
|
|
"loss/base_kto": 3.0540947914123535,
|
|
"metrics/advantage_var": 389.7168884277344,
|
|
"regularization/mmd": 1.1566543579101562,
|
|
"regularization/rkhs_norm": 222.8621063232422,
|
|
"rewards/chosen": 0.618285902664852,
|
|
"rewards/margins": 0.9646606616897455,
|
|
"rewards/rejected": -0.3463747590248936,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 14.874004364013672,
|
|
"kl": 15.622693061828613,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33923773.896713614,
|
|
"logits/rejected": -35869502.70202808,
|
|
"logps/chosen": -493.13478090766824,
|
|
"logps/rejected": -407.02525351014043,
|
|
"loss": 0.529,
|
|
"loss/base_kto": 2.9775712490081787,
|
|
"metrics/advantage_var": 467.7118225097656,
|
|
"regularization/mmd": 1.2547889947891235,
|
|
"regularization/rkhs_norm": 241.7705078125,
|
|
"rewards/chosen": 0.6382726152924492,
|
|
"rewards/margins": 1.075445539221125,
|
|
"rewards/rejected": -0.4371729239286759,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 14.977208137512207,
|
|
"kl": 9.718948364257812,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34752083.34883721,
|
|
"logits/rejected": -35523466.28031496,
|
|
"logps/chosen": -456.39171511627904,
|
|
"logps/rejected": -402.3578494094488,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.1442782878875732,
|
|
"metrics/advantage_var": 351.5884704589844,
|
|
"regularization/mmd": 1.0862476825714111,
|
|
"regularization/rkhs_norm": 209.2962646484375,
|
|
"rewards/chosen": 0.49977743459302326,
|
|
"rewards/margins": 0.8968435659445242,
|
|
"rewards/rejected": -0.397066131351501,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 10.490389823913574,
|
|
"kl": 14.246256828308105,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34071342.6918239,
|
|
"logits/rejected": -35946155.72670808,
|
|
"logps/chosen": -485.077731918239,
|
|
"logps/rejected": -368.78188082298135,
|
|
"loss": 0.5234,
|
|
"loss/base_kto": 3.106316089630127,
|
|
"metrics/advantage_var": 342.7874450683594,
|
|
"regularization/mmd": 1.0810221433639526,
|
|
"regularization/rkhs_norm": 208.28945922851562,
|
|
"rewards/chosen": 0.5692803724756781,
|
|
"rewards/margins": 0.8992850941705296,
|
|
"rewards/rejected": -0.3300047216948515,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 11.920390129089355,
|
|
"kl": 9.458094596862793,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34456400.22677165,
|
|
"logits/rejected": -34348697.20310078,
|
|
"logps/chosen": -477.24379921259845,
|
|
"logps/rejected": -378.56332364341085,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.113023519515991,
|
|
"metrics/advantage_var": 366.575439453125,
|
|
"regularization/mmd": 1.1278514862060547,
|
|
"regularization/rkhs_norm": 217.31240844726562,
|
|
"rewards/chosen": 0.5088973878875492,
|
|
"rewards/margins": 0.9098979386233776,
|
|
"rewards/rejected": -0.40100055073582846,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 10.451973915100098,
|
|
"kl": 12.80419921875,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35743516.44444445,
|
|
"logits/rejected": -35723599.47581904,
|
|
"logps/chosen": -478.9630281690141,
|
|
"logps/rejected": -372.9942228939158,
|
|
"loss": 0.5261,
|
|
"loss/base_kto": 3.093827486038208,
|
|
"metrics/advantage_var": 354.83294677734375,
|
|
"regularization/mmd": 1.1146318912506104,
|
|
"regularization/rkhs_norm": 214.7652587890625,
|
|
"rewards/chosen": 0.5280838132091158,
|
|
"rewards/margins": 0.9237132675376796,
|
|
"rewards/rejected": -0.3956294543285638,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 9.721475601196289,
|
|
"kl": 14.141409873962402,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34409271.550834596,
|
|
"logits/rejected": -35842974.71175523,
|
|
"logps/chosen": -467.9213770864947,
|
|
"logps/rejected": -361.8383152173913,
|
|
"loss": 0.5254,
|
|
"loss/base_kto": 3.0692737102508545,
|
|
"metrics/advantage_var": 386.54248046875,
|
|
"regularization/mmd": 1.1342560052871704,
|
|
"regularization/rkhs_norm": 218.5464324951172,
|
|
"rewards/chosen": 0.6341614325238287,
|
|
"rewards/margins": 0.9522332248069407,
|
|
"rewards/rejected": -0.31807179228311194,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 12.84123706817627,
|
|
"kl": 11.918646812438965,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35401582.8921095,
|
|
"logits/rejected": -37565171.18057663,
|
|
"logps/chosen": -495.46643518518516,
|
|
"logps/rejected": -387.23937784522,
|
|
"loss": 0.5271,
|
|
"loss/base_kto": 3.0971181392669678,
|
|
"metrics/advantage_var": 363.2221374511719,
|
|
"regularization/mmd": 1.1196831464767456,
|
|
"regularization/rkhs_norm": 215.73855590820312,
|
|
"rewards/chosen": 0.5170429616734602,
|
|
"rewards/margins": 0.9170634857866999,
|
|
"rewards/rejected": -0.40002052411323974,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 12.777566909790039,
|
|
"kl": 9.683692932128906,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34964963.018867925,
|
|
"logits/rejected": -36498492.42236025,
|
|
"logps/chosen": -475.05139544025155,
|
|
"logps/rejected": -372.8525815217391,
|
|
"loss": 0.53,
|
|
"loss/base_kto": 3.1250438690185547,
|
|
"metrics/advantage_var": 349.35504150390625,
|
|
"regularization/mmd": 1.115165114402771,
|
|
"regularization/rkhs_norm": 214.8680419921875,
|
|
"rewards/chosen": 0.47042045353343653,
|
|
"rewards/margins": 0.8982806898494303,
|
|
"rewards/rejected": -0.4278602363159938,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 13.83116626739502,
|
|
"kl": 12.920928001403809,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -36302362.52433281,
|
|
"logits/rejected": -36534147.7822706,
|
|
"logps/chosen": -489.30170722135006,
|
|
"logps/rejected": -389.0027702177294,
|
|
"loss": 0.5268,
|
|
"loss/base_kto": 3.1076176166534424,
|
|
"metrics/advantage_var": 348.1242370605469,
|
|
"regularization/mmd": 1.1069954633712769,
|
|
"regularization/rkhs_norm": 213.2938995361328,
|
|
"rewards/chosen": 0.5731726246688579,
|
|
"rewards/margins": 0.9055391883383028,
|
|
"rewards/rejected": -0.332366563669445,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 10.138191223144531,
|
|
"kl": 10.839698791503906,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34733122.37037037,
|
|
"logits/rejected": -36802955.341772154,
|
|
"logps/chosen": -471.93161651234567,
|
|
"logps/rejected": -346.95475672468353,
|
|
"loss": 0.5226,
|
|
"loss/base_kto": 3.072765588760376,
|
|
"metrics/advantage_var": 362.6602478027344,
|
|
"regularization/mmd": 1.1081881523132324,
|
|
"regularization/rkhs_norm": 213.5237274169922,
|
|
"rewards/chosen": 0.547323203381197,
|
|
"rewards/margins": 0.9454841861167463,
|
|
"rewards/rejected": -0.39816098273554934,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 10.297775268554688,
|
|
"kl": 12.261195182800293,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34236499.27710843,
|
|
"logits/rejected": -35840422.233766235,
|
|
"logps/chosen": -483.4094503012048,
|
|
"logps/rejected": -389.18633319805195,
|
|
"loss": 0.5246,
|
|
"loss/base_kto": 3.023761510848999,
|
|
"metrics/advantage_var": 406.6839599609375,
|
|
"regularization/mmd": 1.173167109489441,
|
|
"regularization/rkhs_norm": 226.0437774658203,
|
|
"rewards/chosen": 0.6181016944977175,
|
|
"rewards/margins": 1.0205599101864475,
|
|
"rewards/rejected": -0.4024582156887302,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 13.980240821838379,
|
|
"kl": 14.290509223937988,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -33901226.666666664,
|
|
"logits/rejected": -34959430.62068965,
|
|
"logps/chosen": -476.6505062305296,
|
|
"logps/rejected": -362.0511363636364,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.062887668609619,
|
|
"metrics/advantage_var": 357.87786865234375,
|
|
"regularization/mmd": 1.1319570541381836,
|
|
"regularization/rkhs_norm": 218.1034698486328,
|
|
"rewards/chosen": 0.5935802043784073,
|
|
"rewards/margins": 0.9548575783589863,
|
|
"rewards/rejected": -0.36127737398057896,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 12.550543785095215,
|
|
"kl": 13.003260612487793,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34526914.89032258,
|
|
"logits/rejected": -36702536.92121212,
|
|
"logps/chosen": -493.12318548387094,
|
|
"logps/rejected": -375.2939393939394,
|
|
"loss": 0.527,
|
|
"loss/base_kto": 3.0775673389434814,
|
|
"metrics/advantage_var": 368.90826416015625,
|
|
"regularization/mmd": 1.1387715339660645,
|
|
"regularization/rkhs_norm": 219.41650390625,
|
|
"rewards/chosen": 0.5661862281060988,
|
|
"rewards/margins": 0.9272098709061698,
|
|
"rewards/rejected": -0.36102364280007104,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 12.30187702178955,
|
|
"kl": 7.758034706115723,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34263442.67895545,
|
|
"logits/rejected": -35832615.47853736,
|
|
"logps/chosen": -491.775153609831,
|
|
"logps/rejected": -387.5150536565978,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.1168315410614014,
|
|
"metrics/advantage_var": 368.3890075683594,
|
|
"regularization/mmd": 1.1242722272872925,
|
|
"regularization/rkhs_norm": 216.6227569580078,
|
|
"rewards/chosen": 0.47406005859375,
|
|
"rewards/margins": 0.9331607788280007,
|
|
"rewards/rejected": -0.4591007202342508,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 8.568299293518066,
|
|
"kl": 13.640970230102539,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33591270.59844961,
|
|
"logits/rejected": -37399105.31023622,
|
|
"logps/chosen": -493.2200581395349,
|
|
"logps/rejected": -351.5467765748032,
|
|
"loss": 0.5273,
|
|
"loss/base_kto": 3.0856311321258545,
|
|
"metrics/advantage_var": 387.4796447753906,
|
|
"regularization/mmd": 1.1325229406356812,
|
|
"regularization/rkhs_norm": 218.21255493164062,
|
|
"rewards/chosen": 0.5919730726138566,
|
|
"rewards/margins": 0.9574057397829019,
|
|
"rewards/rejected": -0.36543266716904527,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 7.520174503326416,
|
|
"kl": 14.801548957824707,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -33570673.73182552,
|
|
"logits/rejected": -35778814.06354009,
|
|
"logps/chosen": -461.5297859450727,
|
|
"logps/rejected": -372.69095121028744,
|
|
"loss": 0.523,
|
|
"loss/base_kto": 3.0906670093536377,
|
|
"metrics/advantage_var": 337.0397644042969,
|
|
"regularization/mmd": 1.0931990146636963,
|
|
"regularization/rkhs_norm": 210.63565063476562,
|
|
"rewards/chosen": 0.5426389235094407,
|
|
"rewards/margins": 0.91538383145993,
|
|
"rewards/rejected": -0.3727449079504893,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.153304100036621,
|
|
"kl": 12.090093612670898,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34274220.972972974,
|
|
"logits/rejected": -35502780.456026055,
|
|
"logps/chosen": -475.06465840840843,
|
|
"logps/rejected": -389.267457247557,
|
|
"loss": 0.5308,
|
|
"loss/base_kto": 3.1084365844726562,
|
|
"metrics/advantage_var": 365.2424011230469,
|
|
"regularization/mmd": 1.1381527185440063,
|
|
"regularization/rkhs_norm": 219.29727172851562,
|
|
"rewards/chosen": 0.5549318239137575,
|
|
"rewards/margins": 0.9265550091616199,
|
|
"rewards/rejected": -0.37162318524786236,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 10.674633026123047,
|
|
"kl": 13.322845458984375,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35482065.9116203,
|
|
"logits/rejected": -35838371.396113604,
|
|
"logps/chosen": -488.06106792144027,
|
|
"logps/rejected": -369.5948477204783,
|
|
"loss": 0.5287,
|
|
"loss/base_kto": 3.0646259784698486,
|
|
"metrics/advantage_var": 399.35321044921875,
|
|
"regularization/mmd": 1.165199637413025,
|
|
"regularization/rkhs_norm": 224.5085906982422,
|
|
"rewards/chosen": 0.581476339536876,
|
|
"rewards/margins": 0.955891533331252,
|
|
"rewards/rejected": -0.37441519379437593,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 8.666868209838867,
|
|
"kl": 11.906843185424805,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35078330.46433991,
|
|
"logits/rejected": -35586740.56038647,
|
|
"logps/chosen": -483.5044575113809,
|
|
"logps/rejected": -387.401519726248,
|
|
"loss": 0.5216,
|
|
"loss/base_kto": 3.0373425483703613,
|
|
"metrics/advantage_var": 371.4295959472656,
|
|
"regularization/mmd": 1.1354891061782837,
|
|
"regularization/rkhs_norm": 218.7840118408203,
|
|
"rewards/chosen": 0.5717829934382113,
|
|
"rewards/margins": 0.9905606312577564,
|
|
"rewards/rejected": -0.4187776378195451,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 11.692424774169922,
|
|
"kl": 13.390151977539062,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34022912.79379845,
|
|
"logits/rejected": -35373110.82834645,
|
|
"logps/chosen": -482.69447674418603,
|
|
"logps/rejected": -376.36097440944883,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.034217596054077,
|
|
"metrics/advantage_var": 392.9123229980469,
|
|
"regularization/mmd": 1.1644567251205444,
|
|
"regularization/rkhs_norm": 224.365478515625,
|
|
"rewards/chosen": 0.5863914312318314,
|
|
"rewards/margins": 0.9843042691839721,
|
|
"rewards/rejected": -0.3979128379521408,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 9.955215454101562,
|
|
"kl": 14.835789680480957,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -36104574.593406595,
|
|
"logits/rejected": -37224244.155521,
|
|
"logps/chosen": -485.3425235478807,
|
|
"logps/rejected": -370.96359836702953,
|
|
"loss": 0.5249,
|
|
"loss/base_kto": 3.0935723781585693,
|
|
"metrics/advantage_var": 345.5436706542969,
|
|
"regularization/mmd": 1.1055536270141602,
|
|
"regularization/rkhs_norm": 213.01611328125,
|
|
"rewards/chosen": 0.6026077629819957,
|
|
"rewards/margins": 0.9071907533287589,
|
|
"rewards/rejected": -0.3045829903467632,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 8.96081829071045,
|
|
"kl": 13.66186809539795,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33680170.267912775,
|
|
"logits/rejected": -36078116.9153605,
|
|
"logps/chosen": -460.74591121495325,
|
|
"logps/rejected": -376.73474235893417,
|
|
"loss": 0.5207,
|
|
"loss/base_kto": 3.078190803527832,
|
|
"metrics/advantage_var": 344.8306884765625,
|
|
"regularization/mmd": 1.0873230695724487,
|
|
"regularization/rkhs_norm": 209.5034637451172,
|
|
"rewards/chosen": 0.5968351275007301,
|
|
"rewards/margins": 0.9276716956335671,
|
|
"rewards/rejected": -0.330836568132837,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 12.003533363342285,
|
|
"kl": 13.155600547790527,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34270780.32628399,
|
|
"logits/rejected": -35165478.93851133,
|
|
"logps/chosen": -477.3462990936556,
|
|
"logps/rejected": -352.68648867313914,
|
|
"loss": 0.5251,
|
|
"loss/base_kto": 3.098294973373413,
|
|
"metrics/advantage_var": 356.3811950683594,
|
|
"regularization/mmd": 1.1022199392318726,
|
|
"regularization/rkhs_norm": 212.373779296875,
|
|
"rewards/chosen": 0.5785075300049566,
|
|
"rewards/margins": 0.917265535684189,
|
|
"rewards/rejected": -0.3387580056792324,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 10.422096252441406,
|
|
"kl": 13.961255073547363,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34598256.64,
|
|
"logits/rejected": -35445864.74503817,
|
|
"logps/chosen": -478.49345,
|
|
"logps/rejected": -386.29017175572517,
|
|
"loss": 0.5233,
|
|
"loss/base_kto": 3.076866626739502,
|
|
"metrics/advantage_var": 351.24884033203125,
|
|
"regularization/mmd": 1.1095365285873413,
|
|
"regularization/rkhs_norm": 213.78355407714844,
|
|
"rewards/chosen": 0.551825390625,
|
|
"rewards/margins": 0.9326069108569895,
|
|
"rewards/rejected": -0.3807815202319895,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 10.507048606872559,
|
|
"kl": 12.8363037109375,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -36340272.3328,
|
|
"logits/rejected": -36478532.00610687,
|
|
"logps/chosen": -504.8625,
|
|
"logps/rejected": -385.6226860687023,
|
|
"loss": 0.5249,
|
|
"loss/base_kto": 3.0856473445892334,
|
|
"metrics/advantage_var": 352.6623229980469,
|
|
"regularization/mmd": 1.1136070489883423,
|
|
"regularization/rkhs_norm": 214.5678253173828,
|
|
"rewards/chosen": 0.567965234375,
|
|
"rewards/margins": 0.9208772403163765,
|
|
"rewards/rejected": -0.35291200594137645,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 11.083954811096191,
|
|
"kl": 14.43707275390625,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35108926.86268175,
|
|
"logits/rejected": -36068589.02269289,
|
|
"logps/chosen": -479.93245153473345,
|
|
"logps/rejected": -380.5613180786687,
|
|
"loss": 0.5238,
|
|
"loss/base_kto": 3.059920072555542,
|
|
"metrics/advantage_var": 361.0622253417969,
|
|
"regularization/mmd": 1.1302847862243652,
|
|
"regularization/rkhs_norm": 217.78125,
|
|
"rewards/chosen": 0.5592205682363186,
|
|
"rewards/margins": 0.9417497072140157,
|
|
"rewards/rejected": -0.38252913897769714,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 9.955196380615234,
|
|
"kl": 12.403254508972168,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35138235.39356815,
|
|
"logits/rejected": -35390361.11004785,
|
|
"logps/chosen": -489.0685777182236,
|
|
"logps/rejected": -396.19659589314193,
|
|
"loss": 0.5287,
|
|
"loss/base_kto": 3.0336174964904785,
|
|
"metrics/advantage_var": 407.16680908203125,
|
|
"regularization/mmd": 1.1960941553115845,
|
|
"regularization/rkhs_norm": 230.46128845214844,
|
|
"rewards/chosen": 0.5870492184508997,
|
|
"rewards/margins": 0.9914722279065791,
|
|
"rewards/rejected": -0.4044230094556793,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 11.344730377197266,
|
|
"kl": 14.39224910736084,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34632143.82137161,
|
|
"logits/rejected": -34399562.87901991,
|
|
"logps/chosen": -467.9655602073365,
|
|
"logps/rejected": -368.33580589586524,
|
|
"loss": 0.5236,
|
|
"loss/base_kto": 3.100672483444214,
|
|
"metrics/advantage_var": 348.1331481933594,
|
|
"regularization/mmd": 1.08786141872406,
|
|
"regularization/rkhs_norm": 209.6072235107422,
|
|
"rewards/chosen": 0.5287956468986742,
|
|
"rewards/margins": 0.8986455134999376,
|
|
"rewards/rejected": -0.3698498666012634,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 12.729537010192871,
|
|
"kl": 13.068961143493652,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35895873.163636364,
|
|
"logits/rejected": -35758020.54193548,
|
|
"logps/chosen": -483.4882102272727,
|
|
"logps/rejected": -372.08689516129033,
|
|
"loss": 0.5269,
|
|
"loss/base_kto": 3.126361846923828,
|
|
"metrics/advantage_var": 338.41412353515625,
|
|
"regularization/mmd": 1.0887268781661987,
|
|
"regularization/rkhs_norm": 209.7739715576172,
|
|
"rewards/chosen": 0.5410943233605587,
|
|
"rewards/margins": 0.8966187925515869,
|
|
"rewards/rejected": -0.3555244691910282,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 14.17967414855957,
|
|
"kl": 11.574604988098145,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34513269.5345912,
|
|
"logits/rejected": -34806147.97515528,
|
|
"logps/chosen": -478.07881289308176,
|
|
"logps/rejected": -377.5923913043478,
|
|
"loss": 0.531,
|
|
"loss/base_kto": 3.090505361557007,
|
|
"metrics/advantage_var": 388.46661376953125,
|
|
"regularization/mmd": 1.1578208208084106,
|
|
"regularization/rkhs_norm": 223.0868682861328,
|
|
"rewards/chosen": 0.5807230247641509,
|
|
"rewards/margins": 0.9314504614695567,
|
|
"rewards/rejected": -0.3507274367054057,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 8.735228538513184,
|
|
"kl": 12.658534049987793,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34755086.44514106,
|
|
"logits/rejected": -35124801.39563863,
|
|
"logps/chosen": -485.3395865987461,
|
|
"logps/rejected": -354.87670366043614,
|
|
"loss": 0.5251,
|
|
"loss/base_kto": 3.1163642406463623,
|
|
"metrics/advantage_var": 325.663330078125,
|
|
"regularization/mmd": 1.0844676494598389,
|
|
"regularization/rkhs_norm": 208.9532928466797,
|
|
"rewards/chosen": 0.5551200555783454,
|
|
"rewards/margins": 0.8920120142250305,
|
|
"rewards/rejected": -0.33689195864668514,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 10.3009672164917,
|
|
"kl": 13.03227424621582,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34693307.06141733,
|
|
"logits/rejected": -35954256.17364341,
|
|
"logps/chosen": -485.29675196850394,
|
|
"logps/rejected": -381.8583091085271,
|
|
"loss": 0.5325,
|
|
"loss/base_kto": 3.098832368850708,
|
|
"metrics/advantage_var": 422.102294921875,
|
|
"regularization/mmd": 1.160827398300171,
|
|
"regularization/rkhs_norm": 223.66616821289062,
|
|
"rewards/chosen": 0.5834943636195866,
|
|
"rewards/margins": 0.93468535350573,
|
|
"rewards/rejected": -0.3511909898861434,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_loss": 0.5577637589245469,
|
|
"train_runtime": 11071.1673,
|
|
"train_samples_per_second": 13.388,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|