b6b024673c
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed42 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 24.061288833618164,
|
|
"kl": 15.30693244934082,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36756318.64242424,
|
|
"logits/rejected": -37470459.04516129,
|
|
"logps/chosen": -477.7164772727273,
|
|
"logps/rejected": -362.99314516129033,
|
|
"loss": 0.6014,
|
|
"loss/base_kto": 3.946857452392578,
|
|
"metrics/advantage_var": 178.61863708496094,
|
|
"regularization/lipschitz_norm": 876.2899780273438,
|
|
"regularization/mmd": 0.1283591240644455,
|
|
"regularization/rkhs_norm": 152.80850219726562,
|
|
"regularization/w1": 0.7360835671424866,
|
|
"rewards/chosen": 0.19979370579575048,
|
|
"rewards/margins": 0.03685436342002593,
|
|
"rewards/rejected": 0.16293934237572455,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 19.52281951904297,
|
|
"kl": 9.385969161987305,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37766296.44511278,
|
|
"logits/rejected": -39036331.91544715,
|
|
"logps/chosen": -497.0341635338346,
|
|
"logps/rejected": -395.4654471544715,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.8942172527313232,
|
|
"metrics/advantage_var": 163.17442321777344,
|
|
"regularization/lipschitz_norm": 867.9848022460938,
|
|
"regularization/mmd": 0.12266204506158829,
|
|
"regularization/rkhs_norm": 146.0262451171875,
|
|
"regularization/w1": 0.7291072010993958,
|
|
"rewards/chosen": 0.15460649074468397,
|
|
"rewards/margins": 0.09140411503806557,
|
|
"rewards/rejected": 0.0632023757066184,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 20.192529678344727,
|
|
"kl": 18.184659957885742,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34588705.57377049,
|
|
"logits/rejected": -37115717.54029851,
|
|
"logps/chosen": -476.8985655737705,
|
|
"logps/rejected": -353.28507462686565,
|
|
"loss": 0.5883,
|
|
"loss/base_kto": 3.8471505641937256,
|
|
"metrics/advantage_var": 173.1672821044922,
|
|
"regularization/lipschitz_norm": 871.5046997070312,
|
|
"regularization/mmd": 0.12723729014396667,
|
|
"regularization/rkhs_norm": 151.4729461669922,
|
|
"regularization/w1": 0.732063889503479,
|
|
"rewards/chosen": 0.3061334328573258,
|
|
"rewards/margins": 0.15448646293399323,
|
|
"rewards/rejected": 0.15164696992333257,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 20.085376739501953,
|
|
"kl": 3.370481252670288,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35262568.523126,
|
|
"logits/rejected": -37553119.06891271,
|
|
"logps/chosen": -464.0496909888357,
|
|
"logps/rejected": -359.31939127105665,
|
|
"loss": 0.5964,
|
|
"loss/base_kto": 3.866413116455078,
|
|
"metrics/advantage_var": 184.6261444091797,
|
|
"regularization/lipschitz_norm": 920.6685791015625,
|
|
"regularization/mmd": 0.13150206208229065,
|
|
"regularization/rkhs_norm": 156.55007934570312,
|
|
"regularization/w1": 0.7733615636825562,
|
|
"rewards/chosen": -0.05208843176444752,
|
|
"rewards/margins": 0.12107681218176228,
|
|
"rewards/rejected": -0.1731652439462098,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 22.23554229736328,
|
|
"kl": 8.579002380371094,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36692160.30046948,
|
|
"logits/rejected": -37635862.764430575,
|
|
"logps/chosen": -484.7162558685446,
|
|
"logps/rejected": -375.3727574102964,
|
|
"loss": 0.5959,
|
|
"loss/base_kto": 3.812727451324463,
|
|
"metrics/advantage_var": 216.3206329345703,
|
|
"regularization/lipschitz_norm": 972.4998168945312,
|
|
"regularization/mmd": 0.13731805980205536,
|
|
"regularization/rkhs_norm": 163.47389221191406,
|
|
"regularization/w1": 0.8168997764587402,
|
|
"rewards/chosen": 0.164737946177499,
|
|
"rewards/margins": 0.19311716331639942,
|
|
"rewards/rejected": -0.0283792171389004,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 23.748491287231445,
|
|
"kl": 7.484068393707275,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36535573.08235294,
|
|
"logits/rejected": -38160766.29333334,
|
|
"logps/chosen": -497.53161764705885,
|
|
"logps/rejected": -361.0922395833333,
|
|
"loss": 0.5947,
|
|
"loss/base_kto": 3.8688125610351562,
|
|
"metrics/advantage_var": 203.99609375,
|
|
"regularization/lipschitz_norm": 898.7174072265625,
|
|
"regularization/mmd": 0.13416998088359833,
|
|
"regularization/rkhs_norm": 159.72618103027344,
|
|
"regularization/w1": 0.7549225687980652,
|
|
"rewards/chosen": 0.10815998526180491,
|
|
"rewards/margins": 0.11622950446371938,
|
|
"rewards/rejected": -0.00806951920191447,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 26.647647857666016,
|
|
"kl": 10.49285888671875,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36091924.27722772,
|
|
"logits/rejected": -36790928.33234421,
|
|
"logps/chosen": -490.02516501650166,
|
|
"logps/rejected": -368.49135293026706,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.876896619796753,
|
|
"metrics/advantage_var": 182.90757751464844,
|
|
"regularization/lipschitz_norm": 920.4991455078125,
|
|
"regularization/mmd": 0.13089106976985931,
|
|
"regularization/rkhs_norm": 155.8227081298828,
|
|
"regularization/w1": 0.7732192873954773,
|
|
"rewards/chosen": 0.017669704487614898,
|
|
"rewards/margins": 0.07950088165293215,
|
|
"rewards/rejected": -0.061831177165317254,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 16.89284896850586,
|
|
"kl": 8.803842544555664,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36478452.72246696,
|
|
"logits/rejected": -37851956.567612685,
|
|
"logps/chosen": -475.30263399412627,
|
|
"logps/rejected": -384.0806552587646,
|
|
"loss": 0.5949,
|
|
"loss/base_kto": 3.848097324371338,
|
|
"metrics/advantage_var": 187.6973876953125,
|
|
"regularization/lipschitz_norm": 927.83984375,
|
|
"regularization/mmd": 0.13174079358577728,
|
|
"regularization/rkhs_norm": 156.83425903320312,
|
|
"regularization/w1": 0.779385507106781,
|
|
"rewards/chosen": 0.13681265541110268,
|
|
"rewards/margins": 0.15413100207469077,
|
|
"rewards/rejected": -0.017318346663588075,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 14.733221054077148,
|
|
"kl": 11.558862686157227,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35508045.63608087,
|
|
"logits/rejected": -37472522.85086342,
|
|
"logps/chosen": -468.68234836702953,
|
|
"logps/rejected": -361.3554258241758,
|
|
"loss": 0.5926,
|
|
"loss/base_kto": 3.805783987045288,
|
|
"metrics/advantage_var": 211.26126098632812,
|
|
"regularization/lipschitz_norm": 948.3896484375,
|
|
"regularization/mmd": 0.13838918507099152,
|
|
"regularization/rkhs_norm": 164.7490234375,
|
|
"regularization/w1": 0.796647310256958,
|
|
"rewards/chosen": 0.2632976193821078,
|
|
"rewards/margins": 0.17271871074627515,
|
|
"rewards/rejected": 0.09057890863583264,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 19.31886863708496,
|
|
"kl": 11.500679969787598,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37083814.2976,
|
|
"logits/rejected": -38503438.07022901,
|
|
"logps/chosen": -489.0307,
|
|
"logps/rejected": -374.10314885496183,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.807547092437744,
|
|
"metrics/advantage_var": 231.0459442138672,
|
|
"regularization/lipschitz_norm": 986.7574462890625,
|
|
"regularization/mmd": 0.14365534484386444,
|
|
"regularization/rkhs_norm": 171.01828002929688,
|
|
"regularization/w1": 0.8288761973381042,
|
|
"rewards/chosen": 0.1882507080078125,
|
|
"rewards/margins": 0.1789826418345211,
|
|
"rewards/rejected": 0.009268066173291388,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 22.354700088500977,
|
|
"kl": 5.295507907867432,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -38015022.621138215,
|
|
"logits/rejected": -36863935.32631579,
|
|
"logps/chosen": -475.3343495934959,
|
|
"logps/rejected": -363.2661654135338,
|
|
"loss": 0.6002,
|
|
"loss/base_kto": 3.8267617225646973,
|
|
"metrics/advantage_var": 226.3080596923828,
|
|
"regularization/lipschitz_norm": 992.7683715820312,
|
|
"regularization/mmd": 0.14091962575912476,
|
|
"regularization/rkhs_norm": 167.76145935058594,
|
|
"regularization/w1": 0.8339253664016724,
|
|
"rewards/chosen": -0.02130757153518801,
|
|
"rewards/margins": 0.15146371596701477,
|
|
"rewards/rejected": -0.17277128750220277,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 26.735187530517578,
|
|
"kl": 6.700460910797119,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35961399.82554517,
|
|
"logits/rejected": -37781587.46081505,
|
|
"logps/chosen": -470.1547897196262,
|
|
"logps/rejected": -351.7079496473354,
|
|
"loss": 0.5858,
|
|
"loss/base_kto": 3.805454969406128,
|
|
"metrics/advantage_var": 189.4175262451172,
|
|
"regularization/lipschitz_norm": 892.3472900390625,
|
|
"regularization/mmd": 0.13148470222949982,
|
|
"regularization/rkhs_norm": 156.52940368652344,
|
|
"regularization/w1": 0.7495717406272888,
|
|
"rewards/chosen": 0.05488284577461789,
|
|
"rewards/margins": 0.18783451645417068,
|
|
"rewards/rejected": -0.1329516706795528,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 27.31540870666504,
|
|
"kl": 10.56055736541748,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36674574.20030817,
|
|
"logits/rejected": -37436748.67828843,
|
|
"logps/chosen": -487.87851502311247,
|
|
"logps/rejected": -389.64545364500793,
|
|
"loss": 0.5999,
|
|
"loss/base_kto": 3.727017641067505,
|
|
"metrics/advantage_var": 269.7157897949219,
|
|
"regularization/lipschitz_norm": 1094.1197509765625,
|
|
"regularization/mmd": 0.15351979434490204,
|
|
"regularization/rkhs_norm": 182.7616424560547,
|
|
"regularization/w1": 0.9190606474876404,
|
|
"rewards/chosen": 0.20194508886116863,
|
|
"rewards/margins": 0.25375466879807124,
|
|
"rewards/rejected": -0.05180957993690261,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 14.908998489379883,
|
|
"kl": 6.149159908294678,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36140252.8627451,
|
|
"logits/rejected": -36793448.23952096,
|
|
"logps/chosen": -458.9479166666667,
|
|
"logps/rejected": -366.1334440494012,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.7684807777404785,
|
|
"metrics/advantage_var": 327.1993713378906,
|
|
"regularization/lipschitz_norm": 1041.7467041015625,
|
|
"regularization/mmd": 0.14881670475006104,
|
|
"regularization/rkhs_norm": 177.16275024414062,
|
|
"regularization/w1": 0.8750673532485962,
|
|
"rewards/chosen": 0.031634118821885854,
|
|
"rewards/margins": 0.1966205522051194,
|
|
"rewards/rejected": -0.16498643338323354,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 16.479248046875,
|
|
"kl": 5.856367588043213,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36476318.96456087,
|
|
"logits/rejected": -37083178.1933439,
|
|
"logps/chosen": -463.2150423728813,
|
|
"logps/rejected": -352.0474940570523,
|
|
"loss": 0.5844,
|
|
"loss/base_kto": 3.7703139781951904,
|
|
"metrics/advantage_var": 208.1964111328125,
|
|
"regularization/lipschitz_norm": 914.4244384765625,
|
|
"regularization/mmd": 0.13676691055297852,
|
|
"regularization/rkhs_norm": 162.81777954101562,
|
|
"regularization/w1": 0.7681164741516113,
|
|
"rewards/chosen": 0.09952076116952764,
|
|
"rewards/margins": 0.22728015961631348,
|
|
"rewards/rejected": -0.12775939844678585,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 32.82734680175781,
|
|
"kl": 10.232722282409668,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35282652.7706422,
|
|
"logits/rejected": -38788190.87539936,
|
|
"logps/chosen": -496.43797782874617,
|
|
"logps/rejected": -362.930660942492,
|
|
"loss": 0.595,
|
|
"loss/base_kto": 3.744258165359497,
|
|
"metrics/advantage_var": 260.8705749511719,
|
|
"regularization/lipschitz_norm": 1030.0390625,
|
|
"regularization/mmd": 0.15038061141967773,
|
|
"regularization/rkhs_norm": 179.0245361328125,
|
|
"regularization/w1": 0.8652328848838806,
|
|
"rewards/chosen": 0.1935581358930022,
|
|
"rewards/margins": 0.2532027374894323,
|
|
"rewards/rejected": -0.059644601596430084,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 16.718948364257812,
|
|
"kl": 9.728739738464355,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36039599.7492163,
|
|
"logits/rejected": -37158433.4953271,
|
|
"logps/chosen": -514.2290360501568,
|
|
"logps/rejected": -386.7114972741433,
|
|
"loss": 0.5934,
|
|
"loss/base_kto": 3.7328908443450928,
|
|
"metrics/advantage_var": 246.26589965820312,
|
|
"regularization/lipschitz_norm": 1029.8287353515625,
|
|
"regularization/mmd": 0.14938068389892578,
|
|
"regularization/rkhs_norm": 177.83416748046875,
|
|
"regularization/w1": 0.8650561571121216,
|
|
"rewards/chosen": 0.2354756896398658,
|
|
"rewards/margins": 0.2433312446186613,
|
|
"rewards/rejected": -0.007855554978795512,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 27.747406005859375,
|
|
"kl": 5.277867794036865,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35738149.22132472,
|
|
"logits/rejected": -37592443.546142206,
|
|
"logps/chosen": -481.5026252019386,
|
|
"logps/rejected": -381.6180503025719,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.7919318675994873,
|
|
"metrics/advantage_var": 255.345458984375,
|
|
"regularization/lipschitz_norm": 1078.8089599609375,
|
|
"regularization/mmd": 0.15255644917488098,
|
|
"regularization/rkhs_norm": 181.61483764648438,
|
|
"regularization/w1": 0.9061994552612305,
|
|
"rewards/chosen": -0.09880242170540313,
|
|
"rewards/margins": 0.18309703483239473,
|
|
"rewards/rejected": -0.28189945653779785,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 25.721073150634766,
|
|
"kl": 7.260842800140381,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -38003003.07692308,
|
|
"logits/rejected": -38057083.53015873,
|
|
"logps/chosen": -507.90793269230767,
|
|
"logps/rejected": -365.96031746031747,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.756612777709961,
|
|
"metrics/advantage_var": 262.91455078125,
|
|
"regularization/lipschitz_norm": 1039.7421875,
|
|
"regularization/mmd": 0.15234795212745667,
|
|
"regularization/rkhs_norm": 181.36659240722656,
|
|
"regularization/w1": 0.8733833432197571,
|
|
"rewards/chosen": 0.12968339186448316,
|
|
"rewards/margins": 0.2388784955854963,
|
|
"rewards/rejected": -0.10919510372101314,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 18.518983840942383,
|
|
"kl": 18.72420310974121,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35877628.13293052,
|
|
"logits/rejected": -37894892.94498382,
|
|
"logps/chosen": -502.0282760574018,
|
|
"logps/rejected": -387.2093699433657,
|
|
"loss": 0.5952,
|
|
"loss/base_kto": 3.7739078998565674,
|
|
"metrics/advantage_var": 228.4562530517578,
|
|
"regularization/lipschitz_norm": 1001.5964965820312,
|
|
"regularization/mmd": 0.14673535525798798,
|
|
"regularization/rkhs_norm": 174.68495178222656,
|
|
"regularization/w1": 0.8413410186767578,
|
|
"rewards/chosen": 0.3587621129891663,
|
|
"rewards/margins": 0.20213743317683067,
|
|
"rewards/rejected": 0.15662467981233566,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 24.276382446289062,
|
|
"kl": 18.344409942626953,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37246139.814992025,
|
|
"logits/rejected": -37119363.332312405,
|
|
"logps/chosen": -495.2026515151515,
|
|
"logps/rejected": -381.86983154670753,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.727501630783081,
|
|
"metrics/advantage_var": 252.412841796875,
|
|
"regularization/lipschitz_norm": 1057.0489501953125,
|
|
"regularization/mmd": 0.1496998369693756,
|
|
"regularization/rkhs_norm": 178.21409606933594,
|
|
"regularization/w1": 0.887921154499054,
|
|
"rewards/chosen": 0.29819538414573366,
|
|
"rewards/margins": 0.2374983748123086,
|
|
"rewards/rejected": 0.060697009333425055,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 21.506296157836914,
|
|
"kl": 9.530158042907715,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36917623.898734175,
|
|
"logits/rejected": -37859802.074074075,
|
|
"logps/chosen": -476.8665941455696,
|
|
"logps/rejected": -381.9916087962963,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.7536842823028564,
|
|
"metrics/advantage_var": 252.3427734375,
|
|
"regularization/lipschitz_norm": 1045.080322265625,
|
|
"regularization/mmd": 0.1518622636795044,
|
|
"regularization/rkhs_norm": 180.78843688964844,
|
|
"regularization/w1": 0.8778674006462097,
|
|
"rewards/chosen": 0.12927826748618595,
|
|
"rewards/margins": 0.24287676136835346,
|
|
"rewards/rejected": -0.11359849388216749,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 39.9332160949707,
|
|
"kl": 11.19768238067627,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -37606511.58974359,
|
|
"logits/rejected": -38668893.65853658,
|
|
"logps/chosen": -478.27088341346155,
|
|
"logps/rejected": -378.0344178734756,
|
|
"loss": 0.5846,
|
|
"loss/base_kto": 3.6778664588928223,
|
|
"metrics/advantage_var": 231.51548767089844,
|
|
"regularization/lipschitz_norm": 1014.6818237304688,
|
|
"regularization/mmd": 0.14632557332515717,
|
|
"regularization/rkhs_norm": 174.1970977783203,
|
|
"regularization/w1": 0.8523327112197876,
|
|
"rewards/chosen": 0.15520979807927057,
|
|
"rewards/margins": 0.296487378805708,
|
|
"rewards/rejected": -0.14127758072643745,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 22.971567153930664,
|
|
"kl": 5.234420299530029,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36832377.82636656,
|
|
"logits/rejected": -38046231.343465045,
|
|
"logps/chosen": -488.1368066720257,
|
|
"logps/rejected": -363.7127184650456,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.7873291969299316,
|
|
"metrics/advantage_var": 238.6794891357422,
|
|
"regularization/lipschitz_norm": 1027.65478515625,
|
|
"regularization/mmd": 0.14908884465694427,
|
|
"regularization/rkhs_norm": 177.48672485351562,
|
|
"regularization/w1": 0.8632300496101379,
|
|
"rewards/chosen": -0.01698706617692659,
|
|
"rewards/margins": 0.19051233645771437,
|
|
"rewards/rejected": -0.20749940263464095,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 19.600114822387695,
|
|
"kl": 7.043764591217041,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35966055.69620253,
|
|
"logits/rejected": -37380348.83950617,
|
|
"logps/chosen": -503.74179193037975,
|
|
"logps/rejected": -384.6912133487654,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.7388176918029785,
|
|
"metrics/advantage_var": 232.54234313964844,
|
|
"regularization/lipschitz_norm": 1030.90283203125,
|
|
"regularization/mmd": 0.14990782737731934,
|
|
"regularization/rkhs_norm": 178.4617156982422,
|
|
"regularization/w1": 0.8659583926200867,
|
|
"rewards/chosen": 0.0286272930193551,
|
|
"rewards/margins": 0.2373800782193093,
|
|
"rewards/rejected": -0.2087527851999542,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 18.632308959960938,
|
|
"kl": 6.660374641418457,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35586125.15068493,
|
|
"logits/rejected": -37397286.25287356,
|
|
"logps/chosen": -519.4366438356165,
|
|
"logps/rejected": -385.8475664511494,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.7556488513946533,
|
|
"metrics/advantage_var": 248.62608337402344,
|
|
"regularization/lipschitz_norm": 1026.5970458984375,
|
|
"regularization/mmd": 0.15007810294628143,
|
|
"regularization/rkhs_norm": 178.66441345214844,
|
|
"regularization/w1": 0.8623417019844055,
|
|
"rewards/chosen": -0.14926731423155903,
|
|
"rewards/margins": 0.19231150916497025,
|
|
"rewards/rejected": -0.3415788233965293,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 18.439491271972656,
|
|
"kl": 4.2372260093688965,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -33854764.63785395,
|
|
"logits/rejected": -36803111.51395731,
|
|
"logps/chosen": -493.7649031296572,
|
|
"logps/rejected": -367.70994971264366,
|
|
"loss": 0.5991,
|
|
"loss/base_kto": 3.7052762508392334,
|
|
"metrics/advantage_var": 284.0878601074219,
|
|
"regularization/lipschitz_norm": 1102.3619384765625,
|
|
"regularization/mmd": 0.1613570600748062,
|
|
"regularization/rkhs_norm": 192.0917510986328,
|
|
"regularization/w1": 0.9259840846061707,
|
|
"rewards/chosen": 0.06791314069631381,
|
|
"rewards/margins": 0.3172260692452927,
|
|
"rewards/rejected": -0.24931292854897885,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 23.114517211914062,
|
|
"kl": 15.232810974121094,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35812822.570992365,
|
|
"logits/rejected": -35970410.0864,
|
|
"logps/chosen": -472.31583969465646,
|
|
"logps/rejected": -363.541475,
|
|
"loss": 0.5944,
|
|
"loss/base_kto": 3.728731632232666,
|
|
"metrics/advantage_var": 248.55015563964844,
|
|
"regularization/lipschitz_norm": 1041.8992919921875,
|
|
"regularization/mmd": 0.1515728086233139,
|
|
"regularization/rkhs_norm": 180.44383239746094,
|
|
"regularization/w1": 0.875195324420929,
|
|
"rewards/chosen": 0.2817352760839098,
|
|
"rewards/margins": 0.2441054482030504,
|
|
"rewards/rejected": 0.037629827880859376,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.718399047851562,
|
|
"kl": 14.43424129486084,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36347857.31306991,
|
|
"logits/rejected": -36027777.23472669,
|
|
"logps/chosen": -476.92648176291794,
|
|
"logps/rejected": -354.9749296623794,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.7438225746154785,
|
|
"metrics/advantage_var": 246.60643005371094,
|
|
"regularization/lipschitz_norm": 1021.2345581054688,
|
|
"regularization/mmd": 0.1479819118976593,
|
|
"regularization/rkhs_norm": 176.1689453125,
|
|
"regularization/w1": 0.8578371405601501,
|
|
"rewards/chosen": 0.19300052677606738,
|
|
"rewards/margins": 0.2144404087294632,
|
|
"rewards/rejected": -0.021439881953395832,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 16.897863388061523,
|
|
"kl": 16.039783477783203,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -35212516.74962519,
|
|
"logits/rejected": -34703456.887438826,
|
|
"logps/chosen": -481.88582271364317,
|
|
"logps/rejected": -370.56555872756934,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.699596405029297,
|
|
"metrics/advantage_var": 266.072265625,
|
|
"regularization/lipschitz_norm": 1092.605712890625,
|
|
"regularization/mmd": 0.15338386595249176,
|
|
"regularization/rkhs_norm": 182.59983825683594,
|
|
"regularization/w1": 0.9177886843681335,
|
|
"rewards/chosen": 0.3144906886156531,
|
|
"rewards/margins": 0.2839805220802834,
|
|
"rewards/rejected": 0.030510166535369723,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 20.67934799194336,
|
|
"kl": 16.794050216674805,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37088251.38045113,
|
|
"logits/rejected": -38666849.40487805,
|
|
"logps/chosen": -439.3359962406015,
|
|
"logps/rejected": -352.00767276422766,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.7348594665527344,
|
|
"metrics/advantage_var": 235.28746032714844,
|
|
"regularization/lipschitz_norm": 1001.4083862304688,
|
|
"regularization/mmd": 0.1457410603761673,
|
|
"regularization/rkhs_norm": 173.5012664794922,
|
|
"regularization/w1": 0.8411831259727478,
|
|
"rewards/chosen": 0.26139269377055924,
|
|
"rewards/margins": 0.214713381537526,
|
|
"rewards/rejected": 0.04667931223303322,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 16.07982063293457,
|
|
"kl": 26.757160186767578,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37365537.04510109,
|
|
"logits/rejected": -38823096.86656201,
|
|
"logps/chosen": -504.1108087091757,
|
|
"logps/rejected": -378.87264521193094,
|
|
"loss": 0.5861,
|
|
"loss/base_kto": 3.6837024688720703,
|
|
"metrics/advantage_var": 251.2823486328125,
|
|
"regularization/lipschitz_norm": 1016.2388916015625,
|
|
"regularization/mmd": 0.1514957994222641,
|
|
"regularization/rkhs_norm": 180.35214233398438,
|
|
"regularization/w1": 0.8536405563354492,
|
|
"rewards/chosen": 0.41204810253754376,
|
|
"rewards/margins": 0.2667204491467102,
|
|
"rewards/rejected": 0.1453276533908335,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 23.681554794311523,
|
|
"kl": 23.82567596435547,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36746155.30827068,
|
|
"logits/rejected": -36888169.7300813,
|
|
"logps/chosen": -494.35859962406016,
|
|
"logps/rejected": -402.3973577235772,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.706923246383667,
|
|
"metrics/advantage_var": 281.416259765625,
|
|
"regularization/lipschitz_norm": 1078.4508056640625,
|
|
"regularization/mmd": 0.15745438635349274,
|
|
"regularization/rkhs_norm": 187.44569396972656,
|
|
"regularization/w1": 0.9058987498283386,
|
|
"rewards/chosen": 0.4215988273907425,
|
|
"rewards/margins": 0.2695969611043795,
|
|
"rewards/rejected": 0.15200186628636306,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 15.920571327209473,
|
|
"kl": 14.30078125,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36816419.06849315,
|
|
"logits/rejected": -38410781.42528736,
|
|
"logps/chosen": -512.2769691780821,
|
|
"logps/rejected": -377.38115122126436,
|
|
"loss": 0.5893,
|
|
"loss/base_kto": 3.6138551235198975,
|
|
"metrics/advantage_var": 280.6195983886719,
|
|
"regularization/lipschitz_norm": 1120.3621826171875,
|
|
"regularization/mmd": 0.15948601067066193,
|
|
"regularization/rkhs_norm": 189.8643035888672,
|
|
"regularization/w1": 0.9411041140556335,
|
|
"rewards/chosen": 0.19624930865144077,
|
|
"rewards/margins": 0.31996609053073427,
|
|
"rewards/rejected": -0.12371678187929351,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 16.7027530670166,
|
|
"kl": 9.499100685119629,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35804868.22907489,
|
|
"logits/rejected": -37357171.392320536,
|
|
"logps/chosen": -478.90560756240825,
|
|
"logps/rejected": -376.377165066778,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.7567107677459717,
|
|
"metrics/advantage_var": 251.39244079589844,
|
|
"regularization/lipschitz_norm": 1050.978271484375,
|
|
"regularization/mmd": 0.15127399563789368,
|
|
"regularization/rkhs_norm": 180.0880889892578,
|
|
"regularization/w1": 0.8828216791152954,
|
|
"rewards/chosen": 0.1727736328842006,
|
|
"rewards/margins": 0.2364261172494187,
|
|
"rewards/rejected": -0.06365248436521807,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 18.112823486328125,
|
|
"kl": 29.511234283447266,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36545384.763076924,
|
|
"logits/rejected": -37644834.13333333,
|
|
"logps/chosen": -497.7279807692308,
|
|
"logps/rejected": -370.676810515873,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.6782212257385254,
|
|
"metrics/advantage_var": 245.07275390625,
|
|
"regularization/lipschitz_norm": 1061.2689208984375,
|
|
"regularization/mmd": 0.15339969098567963,
|
|
"regularization/rkhs_norm": 182.61866760253906,
|
|
"regularization/w1": 0.891465961933136,
|
|
"rewards/chosen": 0.4361485877403846,
|
|
"rewards/margins": 0.25101808391939007,
|
|
"rewards/rejected": 0.18513050382099455,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 23.36772918701172,
|
|
"kl": 18.090423583984375,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -38020506.94736842,
|
|
"logits/rejected": -38234812.952380955,
|
|
"logps/chosen": -493.9114925986842,
|
|
"logps/rejected": -384.3053385416667,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.748584747314453,
|
|
"metrics/advantage_var": 225.692626953125,
|
|
"regularization/lipschitz_norm": 1012.1201171875,
|
|
"regularization/mmd": 0.14427240192890167,
|
|
"regularization/rkhs_norm": 171.7528533935547,
|
|
"regularization/w1": 0.8501809239387512,
|
|
"rewards/chosen": 0.16830943760118985,
|
|
"rewards/margins": 0.1903797489658633,
|
|
"rewards/rejected": -0.022070311364673433,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 25.477867126464844,
|
|
"kl": 19.837697982788086,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36650772.417177916,
|
|
"logits/rejected": -37934050.64968153,
|
|
"logps/chosen": -456.47042753067484,
|
|
"logps/rejected": -390.1367436305732,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.779940128326416,
|
|
"metrics/advantage_var": 241.1506805419922,
|
|
"regularization/lipschitz_norm": 1019.7799072265625,
|
|
"regularization/mmd": 0.1484539955854416,
|
|
"regularization/rkhs_norm": 176.73095703125,
|
|
"regularization/w1": 0.8566150665283203,
|
|
"rewards/chosen": 0.2858230029147095,
|
|
"rewards/margins": 0.17603883411246032,
|
|
"rewards/rejected": 0.1097841688022492,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 19.113643646240234,
|
|
"kl": 11.566720962524414,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36213732.51006711,
|
|
"logits/rejected": -38418347.91788856,
|
|
"logps/chosen": -495.9947042785235,
|
|
"logps/rejected": -372.6663535557185,
|
|
"loss": 0.6087,
|
|
"loss/base_kto": 3.6384010314941406,
|
|
"metrics/advantage_var": 372.6042175292969,
|
|
"regularization/lipschitz_norm": 1254.179931640625,
|
|
"regularization/mmd": 0.1777164489030838,
|
|
"regularization/rkhs_norm": 211.5671844482422,
|
|
"regularization/w1": 1.0535110235214233,
|
|
"rewards/chosen": 0.0986863462716941,
|
|
"rewards/margins": 0.36265874334601594,
|
|
"rewards/rejected": -0.26397239707432185,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 15.978631973266602,
|
|
"kl": 6.711389064788818,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36266758.74728682,
|
|
"logits/rejected": -37055218.69606299,
|
|
"logps/chosen": -462.6787790697674,
|
|
"logps/rejected": -376.902657480315,
|
|
"loss": 0.6125,
|
|
"loss/base_kto": 3.3662197589874268,
|
|
"metrics/advantage_var": 578.36328125,
|
|
"regularization/lipschitz_norm": 1560.3765869140625,
|
|
"regularization/mmd": 0.2226952612400055,
|
|
"regularization/rkhs_norm": 265.1134338378906,
|
|
"regularization/w1": 1.3107163906097412,
|
|
"rewards/chosen": 0.20524493077004602,
|
|
"rewards/margins": 0.7216619460259516,
|
|
"rewards/rejected": -0.5164170152559056,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 20.392240524291992,
|
|
"kl": 8.189127922058105,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36700344.16124031,
|
|
"logits/rejected": -36608435.401574805,
|
|
"logps/chosen": -491.9096414728682,
|
|
"logps/rejected": -361.09478346456694,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.355142593383789,
|
|
"metrics/advantage_var": 551.27001953125,
|
|
"regularization/lipschitz_norm": 1489.5924072265625,
|
|
"regularization/mmd": 0.21410833299160004,
|
|
"regularization/rkhs_norm": 254.890869140625,
|
|
"regularization/w1": 1.2512577772140503,
|
|
"rewards/chosen": 0.3428360340207122,
|
|
"rewards/margins": 0.718668274808729,
|
|
"rewards/rejected": -0.37583224078801675,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 19.120193481445312,
|
|
"kl": 15.248980522155762,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -34560517.771336555,
|
|
"logits/rejected": -35581588.39453718,
|
|
"logps/chosen": -473.2853260869565,
|
|
"logps/rejected": -327.47003034901365,
|
|
"loss": 0.5955,
|
|
"loss/base_kto": 3.3272531032562256,
|
|
"metrics/advantage_var": 622.5850830078125,
|
|
"regularization/lipschitz_norm": 1453.4874267578125,
|
|
"regularization/mmd": 0.21606115996837616,
|
|
"regularization/rkhs_norm": 257.2156677246094,
|
|
"regularization/w1": 1.220929503440857,
|
|
"rewards/chosen": 0.5113076226914754,
|
|
"rewards/margins": 0.752188655447071,
|
|
"rewards/rejected": -0.2408810327555956,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 21.287208557128906,
|
|
"kl": 8.600638389587402,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34085927.68992248,
|
|
"logits/rejected": -35332999.055118114,
|
|
"logps/chosen": -462.63381782945737,
|
|
"logps/rejected": -354.75088582677165,
|
|
"loss": 0.6023,
|
|
"loss/base_kto": 3.3238277435302734,
|
|
"metrics/advantage_var": 575.845458984375,
|
|
"regularization/lipschitz_norm": 1520.3084716796875,
|
|
"regularization/mmd": 0.21747727692127228,
|
|
"regularization/rkhs_norm": 258.90155029296875,
|
|
"regularization/w1": 1.2770589590072632,
|
|
"rewards/chosen": 0.28023128065952035,
|
|
"rewards/margins": 0.7620155064838314,
|
|
"rewards/rejected": -0.481784225824311,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 20.242515563964844,
|
|
"kl": 12.774662017822266,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34400400.99115044,
|
|
"logits/rejected": -36918908.17275748,
|
|
"logps/chosen": -470.13477138643066,
|
|
"logps/rejected": -350.66144102990035,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.3126914501190186,
|
|
"metrics/advantage_var": 598.7587890625,
|
|
"regularization/lipschitz_norm": 1519.0128173828125,
|
|
"regularization/mmd": 0.22179608047008514,
|
|
"regularization/rkhs_norm": 264.04296875,
|
|
"regularization/w1": 1.2759708166122437,
|
|
"rewards/chosen": 0.5239427954749724,
|
|
"rewards/margins": 0.7635600766568401,
|
|
"rewards/rejected": -0.23961728118186773,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.645580291748047,
|
|
"kl": 16.47385597229004,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36021200.14953271,
|
|
"logits/rejected": -37191464.92789969,
|
|
"logps/chosen": -505.87091121495325,
|
|
"logps/rejected": -391.00183679467085,
|
|
"loss": 0.6182,
|
|
"loss/base_kto": 3.2659356594085693,
|
|
"metrics/advantage_var": 643.62548828125,
|
|
"regularization/lipschitz_norm": 1715.5570068359375,
|
|
"regularization/mmd": 0.23826880753040314,
|
|
"regularization/rkhs_norm": 283.6533508300781,
|
|
"regularization/w1": 1.4410679340362549,
|
|
"rewards/chosen": 0.5829729692215488,
|
|
"rewards/margins": 0.7918738035472616,
|
|
"rewards/rejected": -0.2089008343257127,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 18.20612335205078,
|
|
"kl": 21.56020736694336,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -36491428.939597316,
|
|
"logits/rejected": -37566961.02923977,
|
|
"logps/chosen": -482.3400272651007,
|
|
"logps/rejected": -381.28154696637426,
|
|
"loss": 0.6103,
|
|
"loss/base_kto": 3.2038655281066895,
|
|
"metrics/advantage_var": 720.5587768554688,
|
|
"regularization/lipschitz_norm": 1710.1494140625,
|
|
"regularization/mmd": 0.2420027256011963,
|
|
"regularization/rkhs_norm": 288.0984802246094,
|
|
"regularization/w1": 1.436525583267212,
|
|
"rewards/chosen": 0.6266877859230809,
|
|
"rewards/margins": 0.8908269278758633,
|
|
"rewards/rejected": -0.2641391419527823,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 15.17077350616455,
|
|
"kl": 6.395068645477295,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35463518.52931854,
|
|
"logits/rejected": -37316008.43143298,
|
|
"logps/chosen": -512.3883716323296,
|
|
"logps/rejected": -394.9394260400616,
|
|
"loss": 0.6309,
|
|
"loss/base_kto": 3.2341620922088623,
|
|
"metrics/advantage_var": 831.8263549804688,
|
|
"regularization/lipschitz_norm": 1859.4007568359375,
|
|
"regularization/mmd": 0.2513946294784546,
|
|
"regularization/rkhs_norm": 299.2793273925781,
|
|
"regularization/w1": 1.561896562576294,
|
|
"rewards/chosen": 0.31251184913889163,
|
|
"rewards/margins": 0.8946853891102418,
|
|
"rewards/rejected": -0.5821735399713501,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 18.710487365722656,
|
|
"kl": 7.018948554992676,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34978295.186228484,
|
|
"logits/rejected": -36578097.92199688,
|
|
"logps/chosen": -477.0570226917058,
|
|
"logps/rejected": -377.05650351014043,
|
|
"loss": 0.6165,
|
|
"loss/base_kto": 3.3800530433654785,
|
|
"metrics/advantage_var": 659.56591796875,
|
|
"regularization/lipschitz_norm": 1573.2835693359375,
|
|
"regularization/mmd": 0.23047176003456116,
|
|
"regularization/rkhs_norm": 274.3711242675781,
|
|
"regularization/w1": 1.3215583562850952,
|
|
"rewards/chosen": 0.20912072505562893,
|
|
"rewards/margins": 0.6827324942334273,
|
|
"rewards/rejected": -0.47361176917779835,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 16.47252082824707,
|
|
"kl": 7.658973693847656,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34910622.27732463,
|
|
"logits/rejected": -34550730.266866565,
|
|
"logps/chosen": -482.3106647634584,
|
|
"logps/rejected": -365.6373688155922,
|
|
"loss": 0.6087,
|
|
"loss/base_kto": 3.310258626937866,
|
|
"metrics/advantage_var": 619.5570678710938,
|
|
"regularization/lipschitz_norm": 1589.989013671875,
|
|
"regularization/mmd": 0.22408504784107208,
|
|
"regularization/rkhs_norm": 266.76788330078125,
|
|
"regularization/w1": 1.3355907201766968,
|
|
"rewards/chosen": 0.3612807306535481,
|
|
"rewards/margins": 0.7526821216328632,
|
|
"rewards/rejected": -0.39140139097931503,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 20.559755325317383,
|
|
"kl": 5.918070316314697,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -36216524.643185295,
|
|
"logits/rejected": -36081745.65869219,
|
|
"logps/chosen": -488.09963629402756,
|
|
"logps/rejected": -392.21351674641147,
|
|
"loss": 0.6388,
|
|
"loss/base_kto": 3.3580710887908936,
|
|
"metrics/advantage_var": 1062.1617431640625,
|
|
"regularization/lipschitz_norm": 1786.8533935546875,
|
|
"regularization/mmd": 0.2510024607181549,
|
|
"regularization/rkhs_norm": 298.8124694824219,
|
|
"regularization/w1": 1.5009568929672241,
|
|
"rewards/chosen": 0.28723359509595137,
|
|
"rewards/margins": 0.7168854669186189,
|
|
"rewards/rejected": -0.42965187182266745,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 15.331923484802246,
|
|
"kl": 14.689013481140137,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34294788.86846276,
|
|
"logits/rejected": -36086365.8798151,
|
|
"logps/chosen": -499.610984548336,
|
|
"logps/rejected": -372.22106124807397,
|
|
"loss": 0.607,
|
|
"loss/base_kto": 3.281017780303955,
|
|
"metrics/advantage_var": 624.8582153320312,
|
|
"regularization/lipschitz_norm": 1605.1187744140625,
|
|
"regularization/mmd": 0.22659635543823242,
|
|
"regularization/rkhs_norm": 269.757568359375,
|
|
"regularization/w1": 1.3482998609542847,
|
|
"rewards/chosen": 0.47460995536598655,
|
|
"rewards/margins": 0.7646141036876233,
|
|
"rewards/rejected": -0.2900041483216367,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 23.260644912719727,
|
|
"kl": 20.15492057800293,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35090765.10843374,
|
|
"logits/rejected": -35754555.84415584,
|
|
"logps/chosen": -486.6987481174699,
|
|
"logps/rejected": -386.2094155844156,
|
|
"loss": 0.6105,
|
|
"loss/base_kto": 3.334191083908081,
|
|
"metrics/advantage_var": 583.4151611328125,
|
|
"regularization/lipschitz_norm": 1580.743408203125,
|
|
"regularization/mmd": 0.2221425622701645,
|
|
"regularization/rkhs_norm": 264.4554138183594,
|
|
"regularization/w1": 1.3278244733810425,
|
|
"rewards/chosen": 0.595872166645096,
|
|
"rewards/margins": 0.7301816999996199,
|
|
"rewards/rejected": -0.1343095333545239,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 21.935510635375977,
|
|
"kl": 14.327792167663574,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34934612.27329192,
|
|
"logits/rejected": -36156428.88050315,
|
|
"logps/chosen": -506.90639557453414,
|
|
"logps/rejected": -369.01203812893084,
|
|
"loss": 0.6202,
|
|
"loss/base_kto": 3.3050034046173096,
|
|
"metrics/advantage_var": 706.4963989257812,
|
|
"regularization/lipschitz_norm": 1689.296142578125,
|
|
"regularization/mmd": 0.2373594045639038,
|
|
"regularization/rkhs_norm": 282.57073974609375,
|
|
"regularization/w1": 1.4190086126327515,
|
|
"rewards/chosen": 0.5040845219392954,
|
|
"rewards/margins": 0.7696624740063527,
|
|
"rewards/rejected": -0.2655779520670573,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 21.3602352142334,
|
|
"kl": 11.154303550720215,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -33303106.64126984,
|
|
"logits/rejected": -34261099.12615385,
|
|
"logps/chosen": -461.4106150793651,
|
|
"logps/rejected": -378.8542788461538,
|
|
"loss": 0.6011,
|
|
"loss/base_kto": 3.326136827468872,
|
|
"metrics/advantage_var": 575.6968994140625,
|
|
"regularization/lipschitz_norm": 1508.380126953125,
|
|
"regularization/mmd": 0.21530114114284515,
|
|
"regularization/rkhs_norm": 256.3108825683594,
|
|
"regularization/w1": 1.267039179801941,
|
|
"rewards/chosen": 0.36222725520058285,
|
|
"rewards/margins": 0.7261650556813521,
|
|
"rewards/rejected": -0.36393780048076924,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 17.760662078857422,
|
|
"kl": 16.07636833190918,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34713917.9543379,
|
|
"logits/rejected": -34955770.24719101,
|
|
"logps/chosen": -473.66828386605783,
|
|
"logps/rejected": -396.5648073836276,
|
|
"loss": 0.6117,
|
|
"loss/base_kto": 3.2765488624572754,
|
|
"metrics/advantage_var": 644.37841796875,
|
|
"regularization/lipschitz_norm": 1652.5120849609375,
|
|
"regularization/mmd": 0.2290533035993576,
|
|
"regularization/rkhs_norm": 272.6824951171875,
|
|
"regularization/w1": 1.3881102800369263,
|
|
"rewards/chosen": 0.5092502170138888,
|
|
"rewards/margins": 0.8143973421110748,
|
|
"rewards/rejected": -0.305147125097186,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 24.83754539489746,
|
|
"kl": 11.345195770263672,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34959804.93290735,
|
|
"logits/rejected": -35160903.24159022,
|
|
"logps/chosen": -475.51267971246006,
|
|
"logps/rejected": -380.2685397553517,
|
|
"loss": 0.5952,
|
|
"loss/base_kto": 3.2789361476898193,
|
|
"metrics/advantage_var": 535.5349731445312,
|
|
"regularization/lipschitz_norm": 1511.8060302734375,
|
|
"regularization/mmd": 0.21293817460536957,
|
|
"regularization/rkhs_norm": 253.49783325195312,
|
|
"regularization/w1": 1.2699168920516968,
|
|
"rewards/chosen": 0.3473303554157099,
|
|
"rewards/margins": 0.7772668489889132,
|
|
"rewards/rejected": -0.42993649357320335,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.399791717529297,
|
|
"kl": 13.0494384765625,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34631248.49270664,
|
|
"logits/rejected": -35461796.48868778,
|
|
"logps/chosen": -482.8523095623987,
|
|
"logps/rejected": -377.24962292609354,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.2969672679901123,
|
|
"metrics/advantage_var": 612.6229858398438,
|
|
"regularization/lipschitz_norm": 1580.170166015625,
|
|
"regularization/mmd": 0.22611801326274872,
|
|
"regularization/rkhs_norm": 269.1881408691406,
|
|
"regularization/w1": 1.3273428678512573,
|
|
"rewards/chosen": 0.3980836543696819,
|
|
"rewards/margins": 0.7441342562886346,
|
|
"rewards/rejected": -0.3460506019189527,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 17.592403411865234,
|
|
"kl": 9.673773765563965,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35337389.88679245,
|
|
"logits/rejected": -34742093.91304348,
|
|
"logps/chosen": -491.1036753144654,
|
|
"logps/rejected": -369.2687791149068,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.311192750930786,
|
|
"metrics/advantage_var": 524.9568481445312,
|
|
"regularization/lipschitz_norm": 1491.1959228515625,
|
|
"regularization/mmd": 0.21303632855415344,
|
|
"regularization/rkhs_norm": 253.6147003173828,
|
|
"regularization/w1": 1.2526044845581055,
|
|
"rewards/chosen": 0.3585570233423005,
|
|
"rewards/margins": 0.7573716808023888,
|
|
"rewards/rejected": -0.3988146574600883,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 19.176700592041016,
|
|
"kl": 9.840781211853027,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34236701.164556965,
|
|
"logits/rejected": -36144216.493827164,
|
|
"logps/chosen": -491.6054193037975,
|
|
"logps/rejected": -357.8932773919753,
|
|
"loss": 0.6105,
|
|
"loss/base_kto": 3.3385682106018066,
|
|
"metrics/advantage_var": 618.5313110351562,
|
|
"regularization/lipschitz_norm": 1575.9930419921875,
|
|
"regularization/mmd": 0.22152887284755707,
|
|
"regularization/rkhs_norm": 263.724853515625,
|
|
"regularization/w1": 1.3238340616226196,
|
|
"rewards/chosen": 0.35719485222538816,
|
|
"rewards/margins": 0.7126980993333767,
|
|
"rewards/rejected": -0.35550324710798853,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 24.93023681640625,
|
|
"kl": 7.672036647796631,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35017728.0,
|
|
"logits/rejected": -36650911.09148265,
|
|
"logps/chosen": -480.35816563467495,
|
|
"logps/rejected": -366.33970820189273,
|
|
"loss": 0.6083,
|
|
"loss/base_kto": 3.3215484619140625,
|
|
"metrics/advantage_var": 570.0585327148438,
|
|
"regularization/lipschitz_norm": 1577.996337890625,
|
|
"regularization/mmd": 0.2196642905473709,
|
|
"regularization/rkhs_norm": 261.5050964355469,
|
|
"regularization/w1": 1.325516939163208,
|
|
"rewards/chosen": 0.29382458855124083,
|
|
"rewards/margins": 0.7623782940976229,
|
|
"rewards/rejected": -0.4685537055463821,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 16.007081985473633,
|
|
"kl": 12.481958389282227,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -33327152.377952754,
|
|
"logits/rejected": -34454297.79844961,
|
|
"logps/chosen": -487.2919291338583,
|
|
"logps/rejected": -382.8316860465116,
|
|
"loss": 0.6202,
|
|
"loss/base_kto": 3.3565120697021484,
|
|
"metrics/advantage_var": 605.4854736328125,
|
|
"regularization/lipschitz_norm": 1639.3326416015625,
|
|
"regularization/mmd": 0.2278117686510086,
|
|
"regularization/rkhs_norm": 271.2044982910156,
|
|
"regularization/w1": 1.3770393133163452,
|
|
"rewards/chosen": 0.30532793660802166,
|
|
"rewards/margins": 0.702149495360747,
|
|
"rewards/rejected": -0.3968215587527253,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 25.13614845275879,
|
|
"kl": 10.310724258422852,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34472043.95658915,
|
|
"logits/rejected": -35279454.33700787,
|
|
"logps/chosen": -480.9155523255814,
|
|
"logps/rejected": -373.62741141732283,
|
|
"loss": 0.6068,
|
|
"loss/base_kto": 3.2843806743621826,
|
|
"metrics/advantage_var": 593.7132568359375,
|
|
"regularization/lipschitz_norm": 1596.0906982421875,
|
|
"regularization/mmd": 0.22891679406166077,
|
|
"regularization/rkhs_norm": 272.5199890136719,
|
|
"regularization/w1": 1.3407163619995117,
|
|
"rewards/chosen": 0.35151769357134205,
|
|
"rewards/margins": 0.7682584835104415,
|
|
"rewards/rejected": -0.4167407899390994,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.02848243713379,
|
|
"kl": 17.374481201171875,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35234391.7258567,
|
|
"logits/rejected": -35805655.87460815,
|
|
"logps/chosen": -502.4301499221184,
|
|
"logps/rejected": -379.43676528213166,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.2676665782928467,
|
|
"metrics/advantage_var": 601.3102416992188,
|
|
"regularization/lipschitz_norm": 1531.6011962890625,
|
|
"regularization/mmd": 0.22396306693553925,
|
|
"regularization/rkhs_norm": 266.6227111816406,
|
|
"regularization/w1": 1.2865450382232666,
|
|
"rewards/chosen": 0.5784205736772293,
|
|
"rewards/margins": 0.7977082666172028,
|
|
"rewards/rejected": -0.21928769293997355,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 19.544641494750977,
|
|
"kl": 18.450199127197266,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35068594.85457271,
|
|
"logits/rejected": -36597743.29526917,
|
|
"logps/chosen": -476.35410419790105,
|
|
"logps/rejected": -363.16782218597064,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.3154518604278564,
|
|
"metrics/advantage_var": 540.6677856445312,
|
|
"regularization/lipschitz_norm": 1485.3631591796875,
|
|
"regularization/mmd": 0.21378619968891144,
|
|
"regularization/rkhs_norm": 254.5073699951172,
|
|
"regularization/w1": 1.247705101966858,
|
|
"rewards/chosen": 0.5985982228612257,
|
|
"rewards/margins": 0.7273191768974716,
|
|
"rewards/rejected": -0.12872095403624592,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 14.101898193359375,
|
|
"kl": 25.5389461517334,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34941161.87654321,
|
|
"logits/rejected": -35391935.18987342,
|
|
"logps/chosen": -489.872444058642,
|
|
"logps/rejected": -387.87388746044303,
|
|
"loss": 0.6069,
|
|
"loss/base_kto": 3.283374071121216,
|
|
"metrics/advantage_var": 646.7308959960938,
|
|
"regularization/lipschitz_norm": 1602.1788330078125,
|
|
"regularization/mmd": 0.22629740834236145,
|
|
"regularization/rkhs_norm": 269.40167236328125,
|
|
"regularization/w1": 1.3458302021026611,
|
|
"rewards/chosen": 0.6127055132830584,
|
|
"rewards/margins": 0.7432310539850091,
|
|
"rewards/rejected": -0.13052554070195066,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 16.41450309753418,
|
|
"kl": 12.917071342468262,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -35105986.263665594,
|
|
"logits/rejected": -37402636.449848026,
|
|
"logps/chosen": -469.0955084405145,
|
|
"logps/rejected": -390.0357380319149,
|
|
"loss": 0.6167,
|
|
"loss/base_kto": 3.3446907997131348,
|
|
"metrics/advantage_var": 618.182861328125,
|
|
"regularization/lipschitz_norm": 1629.4581298828125,
|
|
"regularization/mmd": 0.22048059105873108,
|
|
"regularization/rkhs_norm": 262.4768981933594,
|
|
"regularization/w1": 1.3687448501586914,
|
|
"rewards/chosen": 0.4144259143105657,
|
|
"rewards/margins": 0.716582936618637,
|
|
"rewards/rejected": -0.30215702230807134,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 18.902488708496094,
|
|
"kl": 8.275690078735352,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34980228.578538105,
|
|
"logits/rejected": -36466507.95604396,
|
|
"logps/chosen": -485.106483281493,
|
|
"logps/rejected": -390.8136773940345,
|
|
"loss": 0.6196,
|
|
"loss/base_kto": 3.3147170543670654,
|
|
"metrics/advantage_var": 657.3211059570312,
|
|
"regularization/lipschitz_norm": 1677.2119140625,
|
|
"regularization/mmd": 0.23319987952709198,
|
|
"regularization/rkhs_norm": 277.618896484375,
|
|
"regularization/w1": 1.4088581800460815,
|
|
"rewards/chosen": 0.3306239767356386,
|
|
"rewards/margins": 0.7948075026475057,
|
|
"rewards/rejected": -0.4641835259118671,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 21.706884384155273,
|
|
"kl": 17.019357681274414,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34272686.23875969,
|
|
"logits/rejected": -35265779.50236221,
|
|
"logps/chosen": -466.8390988372093,
|
|
"logps/rejected": -375.6023622047244,
|
|
"loss": 0.5995,
|
|
"loss/base_kto": 3.3403007984161377,
|
|
"metrics/advantage_var": 513.0325317382812,
|
|
"regularization/lipschitz_norm": 1489.460205078125,
|
|
"regularization/mmd": 0.2047690600156784,
|
|
"regularization/rkhs_norm": 243.77268981933594,
|
|
"regularization/w1": 1.2511465549468994,
|
|
"rewards/chosen": 0.4797371324642684,
|
|
"rewards/margins": 0.6606616396654557,
|
|
"rewards/rejected": -0.18092450720118725,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 20.606470108032227,
|
|
"kl": 9.906996726989746,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35509742.77675841,
|
|
"logits/rejected": -36660744.178913735,
|
|
"logps/chosen": -464.4618214831804,
|
|
"logps/rejected": -402.57413138977637,
|
|
"loss": 0.6111,
|
|
"loss/base_kto": 3.347426652908325,
|
|
"metrics/advantage_var": 557.73095703125,
|
|
"regularization/lipschitz_norm": 1575.1258544921875,
|
|
"regularization/mmd": 0.2184298038482666,
|
|
"regularization/rkhs_norm": 260.03546142578125,
|
|
"regularization/w1": 1.3231056928634644,
|
|
"rewards/chosen": 0.4095702565044438,
|
|
"rewards/margins": 0.7395967063721555,
|
|
"rewards/rejected": -0.3300264498677117,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 20.929731369018555,
|
|
"kl": 13.325279235839844,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -35300819.141993955,
|
|
"logits/rejected": -36888403.676375404,
|
|
"logps/chosen": -491.6428436555891,
|
|
"logps/rejected": -387.5301881067961,
|
|
"loss": 0.6156,
|
|
"loss/base_kto": 3.371952772140503,
|
|
"metrics/advantage_var": 591.1611328125,
|
|
"regularization/lipschitz_norm": 1585.736572265625,
|
|
"regularization/mmd": 0.22106890380382538,
|
|
"regularization/rkhs_norm": 263.17724609375,
|
|
"regularization/w1": 1.3320187330245972,
|
|
"rewards/chosen": 0.4284123077853569,
|
|
"rewards/margins": 0.7092018281477912,
|
|
"rewards/rejected": -0.28078952036243426,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 19.168020248413086,
|
|
"kl": 24.79313850402832,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34675983.73613194,
|
|
"logits/rejected": -37046298.727569334,
|
|
"logps/chosen": -485.2072713643178,
|
|
"logps/rejected": -380.17368474714516,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.2433724403381348,
|
|
"metrics/advantage_var": 628.1931762695312,
|
|
"regularization/lipschitz_norm": 1547.307373046875,
|
|
"regularization/mmd": 0.22901098430156708,
|
|
"regularization/rkhs_norm": 272.6321716308594,
|
|
"regularization/w1": 1.2997381687164307,
|
|
"rewards/chosen": 0.7070827226767475,
|
|
"rewards/margins": 0.7608968934768119,
|
|
"rewards/rejected": -0.05381417080006436,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 19.294036865234375,
|
|
"kl": 18.57269859313965,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34551352.88888889,
|
|
"logits/rejected": -35493497.304615386,
|
|
"logps/chosen": -496.13164682539684,
|
|
"logps/rejected": -384.56822115384614,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.3235702514648438,
|
|
"metrics/advantage_var": 544.8534545898438,
|
|
"regularization/lipschitz_norm": 1527.4329833984375,
|
|
"regularization/mmd": 0.21786895394325256,
|
|
"regularization/rkhs_norm": 259.3677673339844,
|
|
"regularization/w1": 1.2830437421798706,
|
|
"rewards/chosen": 0.5940665108816964,
|
|
"rewards/margins": 0.703149668934581,
|
|
"rewards/rejected": -0.10908315805288461,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 17.769622802734375,
|
|
"kl": 18.305875778198242,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34146110.678466074,
|
|
"logits/rejected": -35203609.51495016,
|
|
"logps/chosen": -472.86771755162243,
|
|
"logps/rejected": -377.8817483388704,
|
|
"loss": 0.6051,
|
|
"loss/base_kto": 3.3327114582061768,
|
|
"metrics/advantage_var": 558.6193237304688,
|
|
"regularization/lipschitz_norm": 1534.943115234375,
|
|
"regularization/mmd": 0.21883177757263184,
|
|
"regularization/rkhs_norm": 260.5140686035156,
|
|
"regularization/w1": 1.289352297782898,
|
|
"rewards/chosen": 0.5489520857819413,
|
|
"rewards/margins": 0.7060017020918328,
|
|
"rewards/rejected": -0.1570496163098915,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 17.7718563079834,
|
|
"kl": 23.075780868530273,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35102958.66456693,
|
|
"logits/rejected": -35600577.68682171,
|
|
"logps/chosen": -484.91643700787404,
|
|
"logps/rejected": -375.53309108527134,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.2393798828125,
|
|
"metrics/advantage_var": 574.3668823242188,
|
|
"regularization/lipschitz_norm": 1593.2049560546875,
|
|
"regularization/mmd": 0.2220333069562912,
|
|
"regularization/rkhs_norm": 264.3254089355469,
|
|
"regularization/w1": 1.338292121887207,
|
|
"rewards/chosen": 0.6905491626168799,
|
|
"rewards/margins": 0.7939753167017879,
|
|
"rewards/rejected": -0.10342615408490795,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 15.457829475402832,
|
|
"kl": 12.52320384979248,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -33102049.377990432,
|
|
"logits/rejected": -34965566.72588055,
|
|
"logps/chosen": -429.6735446570973,
|
|
"logps/rejected": -376.3966309341501,
|
|
"loss": 0.5887,
|
|
"loss/base_kto": 3.3145387172698975,
|
|
"metrics/advantage_var": 505.8414611816406,
|
|
"regularization/lipschitz_norm": 1414.2044677734375,
|
|
"regularization/mmd": 0.20702052116394043,
|
|
"regularization/rkhs_norm": 246.45298767089844,
|
|
"regularization/w1": 1.1879316568374634,
|
|
"rewards/chosen": 0.46009780421401514,
|
|
"rewards/margins": 0.7476684479676043,
|
|
"rewards/rejected": -0.2875706437535892,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 17.53460121154785,
|
|
"kl": 11.304563522338867,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33528204.4928,
|
|
"logits/rejected": -35500178.95572519,
|
|
"logps/chosen": -480.1816,
|
|
"logps/rejected": -364.10090648854964,
|
|
"loss": 0.5787,
|
|
"loss/base_kto": 3.3365864753723145,
|
|
"metrics/advantage_var": 435.8765563964844,
|
|
"regularization/lipschitz_norm": 1309.818359375,
|
|
"regularization/mmd": 0.19297796487808228,
|
|
"regularization/rkhs_norm": 229.73570251464844,
|
|
"regularization/w1": 1.1002473831176758,
|
|
"rewards/chosen": 0.3936514892578125,
|
|
"rewards/margins": 0.7107887400852815,
|
|
"rewards/rejected": -0.31713725082746896,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 20.982736587524414,
|
|
"kl": 16.45754051208496,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34330087.46325879,
|
|
"logits/rejected": -36283514.128440365,
|
|
"logps/chosen": -488.2354233226837,
|
|
"logps/rejected": -365.8191179281346,
|
|
"loss": 0.5874,
|
|
"loss/base_kto": 3.2102324962615967,
|
|
"metrics/advantage_var": 571.2377319335938,
|
|
"regularization/lipschitz_norm": 1511.39208984375,
|
|
"regularization/mmd": 0.21960020065307617,
|
|
"regularization/rkhs_norm": 261.4288024902344,
|
|
"regularization/w1": 1.2695693969726562,
|
|
"rewards/chosen": 0.6090651442067692,
|
|
"rewards/margins": 0.8277040396906903,
|
|
"rewards/rejected": -0.21863889548392107,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 24.38959312438965,
|
|
"kl": 10.066145896911621,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -33685604.39215686,
|
|
"logits/rejected": -36349195.531531535,
|
|
"logps/chosen": -511.3855187908497,
|
|
"logps/rejected": -370.44906625375376,
|
|
"loss": 0.5724,
|
|
"loss/base_kto": 3.1862754821777344,
|
|
"metrics/advantage_var": 471.1880798339844,
|
|
"regularization/lipschitz_norm": 1413.9962158203125,
|
|
"regularization/mmd": 0.20521269738674164,
|
|
"regularization/rkhs_norm": 244.3008270263672,
|
|
"regularization/w1": 1.187756896018982,
|
|
"rewards/chosen": 0.42331710516237747,
|
|
"rewards/margins": 0.8650323800075351,
|
|
"rewards/rejected": -0.44171527484515766,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 16.744922637939453,
|
|
"kl": 6.093294143676758,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34285671.49770291,
|
|
"logits/rejected": -35901518.3923445,
|
|
"logps/chosen": -450.2323411179173,
|
|
"logps/rejected": -358.33816786283893,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.2421271800994873,
|
|
"metrics/advantage_var": 398.6988220214844,
|
|
"regularization/lipschitz_norm": 1302.47216796875,
|
|
"regularization/mmd": 0.18811742961406708,
|
|
"regularization/rkhs_norm": 223.9492950439453,
|
|
"regularization/w1": 1.0940767526626587,
|
|
"rewards/chosen": 0.32462368040683626,
|
|
"rewards/margins": 0.8251295810560089,
|
|
"rewards/rejected": -0.5005059006491727,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.231117248535156,
|
|
"kl": 14.46772289276123,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33527996.588045236,
|
|
"logits/rejected": -36213519.87897126,
|
|
"logps/chosen": -473.9477483844911,
|
|
"logps/rejected": -368.1678328290469,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.1471424102783203,
|
|
"metrics/advantage_var": 468.8351135253906,
|
|
"regularization/lipschitz_norm": 1388.1361083984375,
|
|
"regularization/mmd": 0.20132914185523987,
|
|
"regularization/rkhs_norm": 239.6775665283203,
|
|
"regularization/w1": 1.16603422164917,
|
|
"rewards/chosen": 0.5777252640978393,
|
|
"rewards/margins": 0.9047327020765175,
|
|
"rewards/rejected": -0.32700743797867815,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 16.150474548339844,
|
|
"kl": 13.114829063415527,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -33886970.19062027,
|
|
"logits/rejected": -35522364.794830374,
|
|
"logps/chosen": -484.9406202723147,
|
|
"logps/rejected": -377.06906300484656,
|
|
"loss": 0.5533,
|
|
"loss/base_kto": 3.1854093074798584,
|
|
"metrics/advantage_var": 406.8522644042969,
|
|
"regularization/lipschitz_norm": 1252.0865478515625,
|
|
"regularization/mmd": 0.18890316784381866,
|
|
"regularization/rkhs_norm": 224.8847198486328,
|
|
"regularization/w1": 1.0517528057098389,
|
|
"rewards/chosen": 0.5183056068131855,
|
|
"rewards/margins": 0.848153888599585,
|
|
"rewards/rejected": -0.3298482817863994,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 15.473814010620117,
|
|
"kl": 17.653127670288086,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33017974.815267175,
|
|
"logits/rejected": -35228214.8864,
|
|
"logps/chosen": -479.8187977099237,
|
|
"logps/rejected": -402.57725,
|
|
"loss": 0.5667,
|
|
"loss/base_kto": 3.2326602935791016,
|
|
"metrics/advantage_var": 400.2431640625,
|
|
"regularization/lipschitz_norm": 1319.3643798828125,
|
|
"regularization/mmd": 0.19271209836006165,
|
|
"regularization/rkhs_norm": 229.41917419433594,
|
|
"regularization/w1": 1.1082661151885986,
|
|
"rewards/chosen": 0.5932373512792223,
|
|
"rewards/margins": 0.7835901833104723,
|
|
"rewards/rejected": -0.19035283203125,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.87969398498535,
|
|
"kl": 18.03767967224121,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33777692.31336406,
|
|
"logits/rejected": -33452657.144674085,
|
|
"logps/chosen": -464.07757296466974,
|
|
"logps/rejected": -374.0326659379968,
|
|
"loss": 0.5503,
|
|
"loss/base_kto": 3.2006988525390625,
|
|
"metrics/advantage_var": 371.6629943847656,
|
|
"regularization/lipschitz_norm": 1215.2666015625,
|
|
"regularization/mmd": 0.18069247901439667,
|
|
"regularization/rkhs_norm": 215.11009216308594,
|
|
"regularization/w1": 1.0208238363265991,
|
|
"rewards/chosen": 0.6284561743201564,
|
|
"rewards/margins": 0.8098387583992378,
|
|
"rewards/rejected": -0.18138258407908137,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.629518508911133,
|
|
"kl": 18.80061912536621,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34484123.5443038,
|
|
"logits/rejected": -35300807.11111111,
|
|
"logps/chosen": -485.61931368670884,
|
|
"logps/rejected": -359.957706404321,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.1661229133605957,
|
|
"metrics/advantage_var": 428.8267517089844,
|
|
"regularization/lipschitz_norm": 1296.47216796875,
|
|
"regularization/mmd": 0.19565041363239288,
|
|
"regularization/rkhs_norm": 232.91714477539062,
|
|
"regularization/w1": 1.0890367031097412,
|
|
"rewards/chosen": 0.5882649482051029,
|
|
"rewards/margins": 0.8664850895806093,
|
|
"rewards/rejected": -0.2782201413755064,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 15.930831909179688,
|
|
"kl": 12.853654861450195,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34078057.69544741,
|
|
"logits/rejected": -36082511.228615865,
|
|
"logps/chosen": -466.06490384615387,
|
|
"logps/rejected": -374.1058028771384,
|
|
"loss": 0.5508,
|
|
"loss/base_kto": 3.140216588973999,
|
|
"metrics/advantage_var": 435.0933532714844,
|
|
"regularization/lipschitz_norm": 1276.956298828125,
|
|
"regularization/mmd": 0.19340650737285614,
|
|
"regularization/rkhs_norm": 230.245849609375,
|
|
"regularization/w1": 1.0726432800292969,
|
|
"rewards/chosen": 0.5829016477384223,
|
|
"rewards/margins": 0.9139953483787314,
|
|
"rewards/rejected": -0.3310937006403091,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 16.821569442749023,
|
|
"kl": 7.62105655670166,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -35488667.80762853,
|
|
"logits/rejected": -36658534.47562777,
|
|
"logps/chosen": -485.698175787728,
|
|
"logps/rejected": -380.7929283604136,
|
|
"loss": 0.5658,
|
|
"loss/base_kto": 3.2048003673553467,
|
|
"metrics/advantage_var": 434.58203125,
|
|
"regularization/lipschitz_norm": 1341.75390625,
|
|
"regularization/mmd": 0.19418182969093323,
|
|
"regularization/rkhs_norm": 231.16885375976562,
|
|
"regularization/w1": 1.1270732879638672,
|
|
"rewards/chosen": 0.36956827597040837,
|
|
"rewards/margins": 0.8421052955507029,
|
|
"rewards/rejected": -0.4725370195802945,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 15.676944732666016,
|
|
"kl": 11.570111274719238,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35635880.640949555,
|
|
"logits/rejected": -35923116.35643564,
|
|
"logps/chosen": -483.6896327893175,
|
|
"logps/rejected": -379.38484942244224,
|
|
"loss": 0.5672,
|
|
"loss/base_kto": 3.2221601009368896,
|
|
"metrics/advantage_var": 428.28082275390625,
|
|
"regularization/lipschitz_norm": 1334.6649169921875,
|
|
"regularization/mmd": 0.1941031962633133,
|
|
"regularization/rkhs_norm": 231.0752410888672,
|
|
"regularization/w1": 1.121118426322937,
|
|
"rewards/chosen": 0.4732017630285608,
|
|
"rewards/margins": 0.8385917978206142,
|
|
"rewards/rejected": -0.3653900347920534,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 16.814882278442383,
|
|
"kl": 16.061059951782227,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -32831983.01421801,
|
|
"logits/rejected": -33324321.632148378,
|
|
"logps/chosen": -489.8043542654028,
|
|
"logps/rejected": -380.7750676197836,
|
|
"loss": 0.5513,
|
|
"loss/base_kto": 3.1702046394348145,
|
|
"metrics/advantage_var": 415.2689514160156,
|
|
"regularization/lipschitz_norm": 1248.6136474609375,
|
|
"regularization/mmd": 0.19127409160137177,
|
|
"regularization/rkhs_norm": 227.707275390625,
|
|
"regularization/w1": 1.0488356351852417,
|
|
"rewards/chosen": 0.6030927661094244,
|
|
"rewards/margins": 0.8348279012659763,
|
|
"rewards/rejected": -0.23173513515655186,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 12.883003234863281,
|
|
"kl": 17.076210021972656,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34445181.34609251,
|
|
"logits/rejected": -34563732.189892806,
|
|
"logps/chosen": -483.13631379585325,
|
|
"logps/rejected": -396.7334178790199,
|
|
"loss": 0.5518,
|
|
"loss/base_kto": 3.1647095680236816,
|
|
"metrics/advantage_var": 399.47119140625,
|
|
"regularization/lipschitz_norm": 1262.6822509765625,
|
|
"regularization/mmd": 0.18878653645515442,
|
|
"regularization/rkhs_norm": 224.74588012695312,
|
|
"regularization/w1": 1.0606530904769897,
|
|
"rewards/chosen": 0.5081650340005731,
|
|
"rewards/margins": 0.8531283530867619,
|
|
"rewards/rejected": -0.34496331908618877,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 12.072251319885254,
|
|
"kl": 12.94206428527832,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -33732443.14551084,
|
|
"logits/rejected": -35300174.33438486,
|
|
"logps/chosen": -486.4296633126935,
|
|
"logps/rejected": -377.6941541798107,
|
|
"loss": 0.5583,
|
|
"loss/base_kto": 3.222911834716797,
|
|
"metrics/advantage_var": 407.7722473144531,
|
|
"regularization/lipschitz_norm": 1255.7818603515625,
|
|
"regularization/mmd": 0.18879146873950958,
|
|
"regularization/rkhs_norm": 224.7517547607422,
|
|
"regularization/w1": 1.0548566579818726,
|
|
"rewards/chosen": 0.543099370903275,
|
|
"rewards/margins": 0.8194063896766879,
|
|
"rewards/rejected": -0.27630701877341285,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 15.54910659790039,
|
|
"kl": 13.353614807128906,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35772769.774960384,
|
|
"logits/rejected": -37077389.60862866,
|
|
"logps/chosen": -498.39064976228207,
|
|
"logps/rejected": -369.52077715716484,
|
|
"loss": 0.5632,
|
|
"loss/base_kto": 3.182352066040039,
|
|
"metrics/advantage_var": 432.904052734375,
|
|
"regularization/lipschitz_norm": 1338.9703369140625,
|
|
"regularization/mmd": 0.19866575300693512,
|
|
"regularization/rkhs_norm": 236.5068817138672,
|
|
"regularization/w1": 1.1247349977493286,
|
|
"rewards/chosen": 0.5464045166402536,
|
|
"rewards/margins": 0.8784624048652271,
|
|
"rewards/rejected": -0.3320578882249735,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 20.564775466918945,
|
|
"kl": 11.661320686340332,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -32944747.08496732,
|
|
"logits/rejected": -35643655.66467066,
|
|
"logps/chosen": -455.8532475490196,
|
|
"logps/rejected": -378.85076721556885,
|
|
"loss": 0.5619,
|
|
"loss/base_kto": 3.2213783264160156,
|
|
"metrics/advantage_var": 388.0046081542969,
|
|
"regularization/lipschitz_norm": 1295.4930419921875,
|
|
"regularization/mmd": 0.18541449308395386,
|
|
"regularization/rkhs_norm": 220.73155212402344,
|
|
"regularization/w1": 1.0882141590118408,
|
|
"rewards/chosen": 0.47248920116549226,
|
|
"rewards/margins": 0.8133289069468007,
|
|
"rewards/rejected": -0.34083970578130846,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 17.54700469970703,
|
|
"kl": 11.190667152404785,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34326311.896103896,
|
|
"logits/rejected": -34741180.144578315,
|
|
"logps/chosen": -484.99746347402595,
|
|
"logps/rejected": -382.47084431475906,
|
|
"loss": 0.5557,
|
|
"loss/base_kto": 3.1432552337646484,
|
|
"metrics/advantage_var": 424.0445861816406,
|
|
"regularization/lipschitz_norm": 1316.8843994140625,
|
|
"regularization/mmd": 0.19596289098262787,
|
|
"regularization/rkhs_norm": 233.28916931152344,
|
|
"regularization/w1": 1.1061829328536987,
|
|
"rewards/chosen": 0.48927921443790584,
|
|
"rewards/margins": 0.9105940514225632,
|
|
"rewards/rejected": -0.4213148369846574,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 19.427793502807617,
|
|
"kl": 9.268104553222656,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33830608.47425897,
|
|
"logits/rejected": -35580778.96713615,
|
|
"logps/chosen": -476.53251755070204,
|
|
"logps/rejected": -366.06729264475746,
|
|
"loss": 0.5555,
|
|
"loss/base_kto": 3.2102913856506348,
|
|
"metrics/advantage_var": 397.2903747558594,
|
|
"regularization/lipschitz_norm": 1246.5311279296875,
|
|
"regularization/mmd": 0.1869805008172989,
|
|
"regularization/rkhs_norm": 222.5958251953125,
|
|
"regularization/w1": 1.0470861196517944,
|
|
"rewards/chosen": 0.413074035168438,
|
|
"rewards/margins": 0.8241628144805957,
|
|
"rewards/rejected": -0.41108877931215765,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 16.621356964111328,
|
|
"kl": 9.321684837341309,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34411588.71358429,
|
|
"logits/rejected": -35057155.82660688,
|
|
"logps/chosen": -467.51825900163664,
|
|
"logps/rejected": -373.80061192077727,
|
|
"loss": 0.5517,
|
|
"loss/base_kto": 3.147170305252075,
|
|
"metrics/advantage_var": 410.929443359375,
|
|
"regularization/lipschitz_norm": 1280.960693359375,
|
|
"regularization/mmd": 0.1903020590543747,
|
|
"regularization/rkhs_norm": 226.550048828125,
|
|
"regularization/w1": 1.0760070085525513,
|
|
"rewards/chosen": 0.4403392835451616,
|
|
"rewards/margins": 0.886958063615696,
|
|
"rewards/rejected": -0.4466187800705344,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 13.901752471923828,
|
|
"kl": 10.043939590454102,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33997121.69206843,
|
|
"logits/rejected": -35513554.58712716,
|
|
"logps/chosen": -488.57494167962676,
|
|
"logps/rejected": -381.6866169544741,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.221677541732788,
|
|
"metrics/advantage_var": 412.7735290527344,
|
|
"regularization/lipschitz_norm": 1280.602294921875,
|
|
"regularization/mmd": 0.19151243567466736,
|
|
"regularization/rkhs_norm": 227.990966796875,
|
|
"regularization/w1": 1.075705885887146,
|
|
"rewards/chosen": 0.4254934450330482,
|
|
"rewards/margins": 0.8191984926883784,
|
|
"rewards/rejected": -0.39370504765533015,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 17.917987823486328,
|
|
"kl": 12.19534683227539,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34556649.03099511,
|
|
"logits/rejected": -35139872.623688154,
|
|
"logps/chosen": -472.51203099510604,
|
|
"logps/rejected": -365.3452258245877,
|
|
"loss": 0.5633,
|
|
"loss/base_kto": 3.1732165813446045,
|
|
"metrics/advantage_var": 440.358642578125,
|
|
"regularization/lipschitz_norm": 1348.8330078125,
|
|
"regularization/mmd": 0.1998414844274521,
|
|
"regularization/rkhs_norm": 237.9065399169922,
|
|
"regularization/w1": 1.1330196857452393,
|
|
"rewards/chosen": 0.5041173838480068,
|
|
"rewards/margins": 0.8656737272583328,
|
|
"rewards/rejected": -0.3615563434103261,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 16.038705825805664,
|
|
"kl": 8.044748306274414,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34879443.95698924,
|
|
"logits/rejected": -36427339.70111288,
|
|
"logps/chosen": -454.03168202764977,
|
|
"logps/rejected": -366.30882352941177,
|
|
"loss": 0.5639,
|
|
"loss/base_kto": 3.2381465435028076,
|
|
"metrics/advantage_var": 407.84893798828125,
|
|
"regularization/lipschitz_norm": 1291.3255615234375,
|
|
"regularization/mmd": 0.18864312767982483,
|
|
"regularization/rkhs_norm": 224.5751495361328,
|
|
"regularization/w1": 1.0847134590148926,
|
|
"rewards/chosen": 0.4419856620823733,
|
|
"rewards/margins": 0.8166408474118894,
|
|
"rewards/rejected": -0.3746551853295161,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 14.634725570678711,
|
|
"kl": 9.459915161132812,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35243809.39130435,
|
|
"logits/rejected": -35290681.96226415,
|
|
"logps/chosen": -479.1896350931677,
|
|
"logps/rejected": -391.6245332154088,
|
|
"loss": 0.569,
|
|
"loss/base_kto": 3.2139854431152344,
|
|
"metrics/advantage_var": 756.03515625,
|
|
"regularization/lipschitz_norm": 1348.5595703125,
|
|
"regularization/mmd": 0.20550327003002167,
|
|
"regularization/rkhs_norm": 244.64675903320312,
|
|
"regularization/w1": 1.132789969444275,
|
|
"rewards/chosen": 0.4553101344138199,
|
|
"rewards/margins": 0.7884958701523223,
|
|
"rewards/rejected": -0.33318573573850235,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 16.276470184326172,
|
|
"kl": 9.328352928161621,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34422215.941605836,
|
|
"logits/rejected": -35732554.003361344,
|
|
"logps/chosen": -505.9730383211679,
|
|
"logps/rejected": -361.4629726890756,
|
|
"loss": 0.5628,
|
|
"loss/base_kto": 3.202509641647339,
|
|
"metrics/advantage_var": 419.1310729980469,
|
|
"regularization/lipschitz_norm": 1316.6361083984375,
|
|
"regularization/mmd": 0.19425955414772034,
|
|
"regularization/rkhs_norm": 231.26138305664062,
|
|
"regularization/w1": 1.1059744358062744,
|
|
"rewards/chosen": 0.5190720606894389,
|
|
"rewards/margins": 0.8420494120303659,
|
|
"rewards/rejected": -0.322977351340927,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 13.80461597442627,
|
|
"kl": 12.417572021484375,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -32793040.0,
|
|
"logits/rejected": -34299273.6,
|
|
"logps/chosen": -460.17451171875,
|
|
"logps/rejected": -364.22275390625,
|
|
"loss": 0.5533,
|
|
"loss/base_kto": 3.2077388763427734,
|
|
"metrics/advantage_var": 371.815185546875,
|
|
"regularization/lipschitz_norm": 1230.1234130859375,
|
|
"regularization/mmd": 0.185292050242424,
|
|
"regularization/rkhs_norm": 220.58580017089844,
|
|
"regularization/w1": 1.0333036184310913,
|
|
"rewards/chosen": 0.4350123405456543,
|
|
"rewards/margins": 0.8055490016937256,
|
|
"rewards/rejected": -0.37053666114807127,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 17.276737213134766,
|
|
"kl": 11.258346557617188,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35090397.7108067,
|
|
"logits/rejected": -36635156.545746386,
|
|
"logps/chosen": -492.5710616438356,
|
|
"logps/rejected": -380.5577849117175,
|
|
"loss": 0.5613,
|
|
"loss/base_kto": 3.14424467086792,
|
|
"metrics/advantage_var": 449.4054870605469,
|
|
"regularization/lipschitz_norm": 1363.7271728515625,
|
|
"regularization/mmd": 0.20050835609436035,
|
|
"regularization/rkhs_norm": 238.700439453125,
|
|
"regularization/w1": 1.1455309391021729,
|
|
"rewards/chosen": 0.5235326293759512,
|
|
"rewards/margins": 0.8973924084032134,
|
|
"rewards/rejected": -0.37385977902726225,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 12.550277709960938,
|
|
"kl": 10.444754600524902,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34444704.0,
|
|
"logits/rejected": -35671232.0,
|
|
"logps/chosen": -465.1693359375,
|
|
"logps/rejected": -378.2304931640625,
|
|
"loss": 0.5527,
|
|
"loss/base_kto": 3.1610734462738037,
|
|
"metrics/advantage_var": 407.6515808105469,
|
|
"regularization/lipschitz_norm": 1273.321533203125,
|
|
"regularization/mmd": 0.1907428801059723,
|
|
"regularization/rkhs_norm": 227.07485961914062,
|
|
"regularization/w1": 1.0695899724960327,
|
|
"rewards/chosen": 0.4995744705200195,
|
|
"rewards/margins": 0.8770462989807128,
|
|
"rewards/rejected": -0.3774718284606934,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 16.51201629638672,
|
|
"kl": 13.741698265075684,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -32024102.379585326,
|
|
"logits/rejected": -34391518.284839205,
|
|
"logps/chosen": -478.58542663476874,
|
|
"logps/rejected": -373.9956690275651,
|
|
"loss": 0.5667,
|
|
"loss/base_kto": 3.210426092147827,
|
|
"metrics/advantage_var": 413.5577087402344,
|
|
"regularization/lipschitz_norm": 1346.5155029296875,
|
|
"regularization/mmd": 0.19190925359725952,
|
|
"regularization/rkhs_norm": 228.46337890625,
|
|
"regularization/w1": 1.131072998046875,
|
|
"rewards/chosen": 0.4855345706240032,
|
|
"rewards/margins": 0.8143200682222278,
|
|
"rewards/rejected": -0.32878549759822456,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 18.484439849853516,
|
|
"kl": 11.56452751159668,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34112359.095022626,
|
|
"logits/rejected": -34866048.20745543,
|
|
"logps/chosen": -525.6364064856712,
|
|
"logps/rejected": -385.23429902755265,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.181814670562744,
|
|
"metrics/advantage_var": 416.8466491699219,
|
|
"regularization/lipschitz_norm": 1283.6746826171875,
|
|
"regularization/mmd": 0.19223769009113312,
|
|
"regularization/rkhs_norm": 228.85440063476562,
|
|
"regularization/w1": 1.0782866477966309,
|
|
"rewards/chosen": 0.5313625421998728,
|
|
"rewards/margins": 0.8509247648213021,
|
|
"rewards/rejected": -0.3195622226214293,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 15.074495315551758,
|
|
"kl": 13.301358222961426,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -33762499.88180404,
|
|
"logits/rejected": -35133546.09733124,
|
|
"logps/chosen": -501.5098658631415,
|
|
"logps/rejected": -390.72932201726843,
|
|
"loss": 0.5571,
|
|
"loss/base_kto": 3.155550718307495,
|
|
"metrics/advantage_var": 442.30792236328125,
|
|
"regularization/lipschitz_norm": 1313.21875,
|
|
"regularization/mmd": 0.19832029938697815,
|
|
"regularization/rkhs_norm": 236.09561157226562,
|
|
"regularization/w1": 1.1031038761138916,
|
|
"rewards/chosen": 0.5467796978275661,
|
|
"rewards/margins": 0.8790458421533853,
|
|
"rewards/rejected": -0.33226614432581925,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 17.058185577392578,
|
|
"kl": 12.665131568908691,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34718008.51948052,
|
|
"logits/rejected": -35636825.44578313,
|
|
"logps/chosen": -491.0309456168831,
|
|
"logps/rejected": -374.4232398343373,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.1329848766326904,
|
|
"metrics/advantage_var": 457.51763916015625,
|
|
"regularization/lipschitz_norm": 1389.5159912109375,
|
|
"regularization/mmd": 0.20291300117969513,
|
|
"regularization/rkhs_norm": 241.56309509277344,
|
|
"regularization/w1": 1.1671934127807617,
|
|
"rewards/chosen": 0.5600868324180702,
|
|
"rewards/margins": 0.9178547877844576,
|
|
"rewards/rejected": -0.35776795536638745,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 17.0068302154541,
|
|
"kl": 13.439288139343262,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -33648611.15492958,
|
|
"logits/rejected": -33730553.6099844,
|
|
"logps/chosen": -477.58954420970264,
|
|
"logps/rejected": -391.1602476599064,
|
|
"loss": 0.5589,
|
|
"loss/base_kto": 3.1639182567596436,
|
|
"metrics/advantage_var": 429.63092041015625,
|
|
"regularization/lipschitz_norm": 1322.3231201171875,
|
|
"regularization/mmd": 0.19627168774604797,
|
|
"regularization/rkhs_norm": 233.65676879882812,
|
|
"regularization/w1": 1.1107515096664429,
|
|
"rewards/chosen": 0.5450326743446792,
|
|
"rewards/margins": 0.8872925024331152,
|
|
"rewards/rejected": -0.342259828088436,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 24.721040725708008,
|
|
"kl": 13.245546340942383,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34619685.46341463,
|
|
"logits/rejected": -36175067.897435896,
|
|
"logps/chosen": -466.7677210365854,
|
|
"logps/rejected": -393.8449268830128,
|
|
"loss": 0.5627,
|
|
"loss/base_kto": 3.2227227687835693,
|
|
"metrics/advantage_var": 411.6934509277344,
|
|
"regularization/lipschitz_norm": 1293.9947509765625,
|
|
"regularization/mmd": 0.19191764295101166,
|
|
"regularization/rkhs_norm": 228.473388671875,
|
|
"regularization/w1": 1.0869555473327637,
|
|
"rewards/chosen": 0.5141576906529869,
|
|
"rewards/margins": 0.8208652929338237,
|
|
"rewards/rejected": -0.3067076022808368,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 14.489055633544922,
|
|
"kl": 13.369745254516602,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35949103.10708899,
|
|
"logits/rejected": -36102602.06158833,
|
|
"logps/chosen": -496.85454374057315,
|
|
"logps/rejected": -376.66541734197733,
|
|
"loss": 0.5664,
|
|
"loss/base_kto": 3.215487241744995,
|
|
"metrics/advantage_var": 448.605712890625,
|
|
"regularization/lipschitz_norm": 1330.0577392578125,
|
|
"regularization/mmd": 0.19885368645191193,
|
|
"regularization/rkhs_norm": 236.73057556152344,
|
|
"regularization/w1": 1.11724853515625,
|
|
"rewards/chosen": 0.5273163164003583,
|
|
"rewards/margins": 0.8341436575388054,
|
|
"rewards/rejected": -0.3068273411384471,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 13.536358833312988,
|
|
"kl": 12.86722469329834,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34022955.01812191,
|
|
"logits/rejected": -34775989.44427934,
|
|
"logps/chosen": -480.06558896210873,
|
|
"logps/rejected": -368.3996099554235,
|
|
"loss": 0.551,
|
|
"loss/base_kto": 3.181318998336792,
|
|
"metrics/advantage_var": 385.2022399902344,
|
|
"regularization/lipschitz_norm": 1239.7713623046875,
|
|
"regularization/mmd": 0.18530936539173126,
|
|
"regularization/rkhs_norm": 220.6063690185547,
|
|
"regularization/w1": 1.0414079427719116,
|
|
"rewards/chosen": 0.5009799812731672,
|
|
"rewards/margins": 0.8290710627762319,
|
|
"rewards/rejected": -0.32809108150306465,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 14.650564193725586,
|
|
"kl": 14.825749397277832,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -33724834.47678018,
|
|
"logits/rejected": -34945159.67192429,
|
|
"logps/chosen": -478.2694465944272,
|
|
"logps/rejected": -351.54847693217664,
|
|
"loss": 0.5605,
|
|
"loss/base_kto": 3.2014389038085938,
|
|
"metrics/advantage_var": 422.40191650390625,
|
|
"regularization/lipschitz_norm": 1298.9710693359375,
|
|
"regularization/mmd": 0.19174747169017792,
|
|
"regularization/rkhs_norm": 228.2707977294922,
|
|
"regularization/w1": 1.0911357402801514,
|
|
"rewards/chosen": 0.47620068245997,
|
|
"rewards/margins": 0.8418120987682557,
|
|
"rewards/rejected": -0.3656114163082857,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 15.799490928649902,
|
|
"kl": 10.568368911743164,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34498614.371681415,
|
|
"logits/rejected": -35521988.46511628,
|
|
"logps/chosen": -485.20856379056045,
|
|
"logps/rejected": -390.46776370431894,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.154115676879883,
|
|
"metrics/advantage_var": 479.8663024902344,
|
|
"regularization/lipschitz_norm": 1348.2919921875,
|
|
"regularization/mmd": 0.20181012153625488,
|
|
"regularization/rkhs_norm": 240.25015258789062,
|
|
"regularization/w1": 1.1325653791427612,
|
|
"rewards/chosen": 0.591974849194552,
|
|
"rewards/margins": 0.9252405149522478,
|
|
"rewards/rejected": -0.33326566575769573,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 17.943862915039062,
|
|
"kl": 15.242505073547363,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -32926362.48973144,
|
|
"logits/rejected": -34461483.91962906,
|
|
"logps/chosen": -481.78554502369667,
|
|
"logps/rejected": -367.1816557187017,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.1754376888275146,
|
|
"metrics/advantage_var": 450.428466796875,
|
|
"regularization/lipschitz_norm": 1311.7330322265625,
|
|
"regularization/mmd": 0.19746826589107513,
|
|
"regularization/rkhs_norm": 235.08128356933594,
|
|
"regularization/w1": 1.1018556356430054,
|
|
"rewards/chosen": 0.5447033826397117,
|
|
"rewards/margins": 0.870565742419223,
|
|
"rewards/rejected": -0.3258623597795112,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 15.731083869934082,
|
|
"kl": 13.339346885681152,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34977229.61269841,
|
|
"logits/rejected": -35997368.32,
|
|
"logps/chosen": -483.57232142857146,
|
|
"logps/rejected": -373.6041826923077,
|
|
"loss": 0.5542,
|
|
"loss/base_kto": 3.2174980640411377,
|
|
"metrics/advantage_var": 407.316162109375,
|
|
"regularization/lipschitz_norm": 1221.539306640625,
|
|
"regularization/mmd": 0.19036178290843964,
|
|
"regularization/rkhs_norm": 226.6211395263672,
|
|
"regularization/w1": 1.0260928869247437,
|
|
"rewards/chosen": 0.49627549913194446,
|
|
"rewards/margins": 0.8086349304720887,
|
|
"rewards/rejected": -0.3123594313401442,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.989338484496138e+17,
|
|
"train_loss": 0.587049578961519,
|
|
"train_runtime": 11087.4794,
|
|
"train_samples_per_second": 13.368,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.989338484496138e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|