fe15cc01ec
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed4 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 21.82794952392578,
|
|
"kl": 19.733213424682617,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35289331.512195125,
|
|
"logits/rejected": -36822370.461538464,
|
|
"logps/chosen": -497.5255335365854,
|
|
"logps/rejected": -363.162109375,
|
|
"loss": 0.602,
|
|
"loss/base_kto": 3.8978347778320312,
|
|
"metrics/advantage_var": 197.80735778808594,
|
|
"regularization/lipschitz_norm": 934.49267578125,
|
|
"regularization/mmd": 0.1334814876317978,
|
|
"regularization/rkhs_norm": 158.9065399169922,
|
|
"regularization/w1": 0.7849739193916321,
|
|
"rewards/chosen": 0.3008701510545684,
|
|
"rewards/margins": 0.10311201440311354,
|
|
"rewards/rejected": 0.19775813665145484,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 21.769861221313477,
|
|
"kl": 12.758685111999512,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36458492.86370597,
|
|
"logits/rejected": -36677850.84529506,
|
|
"logps/chosen": -483.7991481623277,
|
|
"logps/rejected": -403.81753389154704,
|
|
"loss": 0.6095,
|
|
"loss/base_kto": 3.9494380950927734,
|
|
"metrics/advantage_var": 194.6944122314453,
|
|
"regularization/lipschitz_norm": 945.7813720703125,
|
|
"regularization/mmd": 0.13222432136535645,
|
|
"regularization/rkhs_norm": 157.409912109375,
|
|
"regularization/w1": 0.7944563031196594,
|
|
"rewards/chosen": 0.13920212376172472,
|
|
"rewards/margins": 0.050537754041703783,
|
|
"rewards/rejected": 0.08866436972002094,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 21.466001510620117,
|
|
"kl": 9.810246467590332,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34970694.56661562,
|
|
"logits/rejected": -36311474.42424242,
|
|
"logps/chosen": -493.6653905053599,
|
|
"logps/rejected": -388.79196570972886,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.8380496501922607,
|
|
"metrics/advantage_var": 189.5191192626953,
|
|
"regularization/lipschitz_norm": 901.5357666015625,
|
|
"regularization/mmd": 0.1319102793931961,
|
|
"regularization/rkhs_norm": 157.03604125976562,
|
|
"regularization/w1": 0.7572900652885437,
|
|
"rewards/chosen": 0.21404192750705878,
|
|
"rewards/margins": 0.14937518554423046,
|
|
"rewards/rejected": 0.06466674196282833,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 23.825450897216797,
|
|
"kl": 4.974595069885254,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37086864.05423729,
|
|
"logits/rejected": -37588707.06086957,
|
|
"logps/chosen": -490.2220868644068,
|
|
"logps/rejected": -393.6825634057971,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.8737030029296875,
|
|
"metrics/advantage_var": 179.11569213867188,
|
|
"regularization/lipschitz_norm": 946.7984619140625,
|
|
"regularization/mmd": 0.12951485812664032,
|
|
"regularization/rkhs_norm": 154.18434143066406,
|
|
"regularization/w1": 0.7953106760978699,
|
|
"rewards/chosen": -0.09145614494711665,
|
|
"rewards/margins": 0.08175607782943453,
|
|
"rewards/rejected": -0.17321222277655118,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 22.79436492919922,
|
|
"kl": 8.591691017150879,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -34779743.403726704,
|
|
"logits/rejected": -36507635.11949685,
|
|
"logps/chosen": -481.51543090062114,
|
|
"logps/rejected": -354.10060436320754,
|
|
"loss": 0.5924,
|
|
"loss/base_kto": 3.8282876014709473,
|
|
"metrics/advantage_var": 202.4976348876953,
|
|
"regularization/lipschitz_norm": 924.4768676757812,
|
|
"regularization/mmd": 0.1344105750322342,
|
|
"regularization/rkhs_norm": 160.0125732421875,
|
|
"regularization/w1": 0.7765604853630066,
|
|
"rewards/chosen": 0.11319585764630241,
|
|
"rewards/margins": 0.16970656101505915,
|
|
"rewards/rejected": -0.05651070336875676,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 26.277545928955078,
|
|
"kl": 14.465189933776855,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36294721.4313099,
|
|
"logits/rejected": -36323259.10703364,
|
|
"logps/chosen": -469.4310103833866,
|
|
"logps/rejected": -374.64623948776756,
|
|
"loss": 0.5983,
|
|
"loss/base_kto": 3.886388063430786,
|
|
"metrics/advantage_var": 193.74246215820312,
|
|
"regularization/lipschitz_norm": 911.8334350585938,
|
|
"regularization/mmd": 0.133758544921875,
|
|
"regularization/rkhs_norm": 159.2363739013672,
|
|
"regularization/w1": 0.7659401297569275,
|
|
"rewards/chosen": 0.22665507648699582,
|
|
"rewards/margins": 0.09204077947760647,
|
|
"rewards/rejected": 0.13461429700938934,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 23.51470947265625,
|
|
"kl": 5.753861904144287,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36272320.28614009,
|
|
"logits/rejected": -37773728.99835797,
|
|
"logps/chosen": -497.83811475409834,
|
|
"logps/rejected": -369.73722290640393,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.8229713439941406,
|
|
"metrics/advantage_var": 214.08824157714844,
|
|
"regularization/lipschitz_norm": 993.9049072265625,
|
|
"regularization/mmd": 0.14072412252426147,
|
|
"regularization/rkhs_norm": 167.5287322998047,
|
|
"regularization/w1": 0.8348800539970398,
|
|
"rewards/chosen": 0.05763329336788899,
|
|
"rewards/margins": 0.19019859597397992,
|
|
"rewards/rejected": -0.13256530260609092,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 15.978097915649414,
|
|
"kl": 14.946639060974121,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35941920.820512824,
|
|
"logits/rejected": -36117766.24390244,
|
|
"logps/chosen": -475.00896434294873,
|
|
"logps/rejected": -348.0709317835366,
|
|
"loss": 0.5874,
|
|
"loss/base_kto": 3.772995710372925,
|
|
"metrics/advantage_var": 203.2220001220703,
|
|
"regularization/lipschitz_norm": 937.8681640625,
|
|
"regularization/mmd": 0.13845016062259674,
|
|
"regularization/rkhs_norm": 164.82164001464844,
|
|
"regularization/w1": 0.7878093123435974,
|
|
"rewards/chosen": 0.30818709349020934,
|
|
"rewards/margins": 0.22307045121279412,
|
|
"rewards/rejected": 0.08511664227741521,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 19.67218780517578,
|
|
"kl": 2.9251906871795654,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35193019.91912908,
|
|
"logits/rejected": -36144486.480376765,
|
|
"logps/chosen": -454.91524105754274,
|
|
"logps/rejected": -361.4044839089482,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.8107666969299316,
|
|
"metrics/advantage_var": 207.85020446777344,
|
|
"regularization/lipschitz_norm": 929.7857666015625,
|
|
"regularization/mmd": 0.13542434573173523,
|
|
"regularization/rkhs_norm": 161.21946716308594,
|
|
"regularization/w1": 0.7810201048851013,
|
|
"rewards/chosen": 0.008817962688012961,
|
|
"rewards/margins": 0.1942005122673399,
|
|
"rewards/rejected": -0.18538254957932693,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 19.236726760864258,
|
|
"kl": 7.678121089935303,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -34794952.01267829,
|
|
"logits/rejected": -37756227.45146379,
|
|
"logps/chosen": -498.29491877971475,
|
|
"logps/rejected": -374.05643297380584,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.821892499923706,
|
|
"metrics/advantage_var": 221.12060546875,
|
|
"regularization/lipschitz_norm": 1006.3059692382812,
|
|
"regularization/mmd": 0.1430058479309082,
|
|
"regularization/rkhs_norm": 170.24505615234375,
|
|
"regularization/w1": 0.8452969789505005,
|
|
"rewards/chosen": 0.05085766485489303,
|
|
"rewards/margins": 0.1549341502756726,
|
|
"rewards/rejected": -0.10407648542077956,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 21.731033325195312,
|
|
"kl": 6.097840785980225,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36237493.18167456,
|
|
"logits/rejected": -36387547.202472955,
|
|
"logps/chosen": -482.0309044233807,
|
|
"logps/rejected": -376.0647942426584,
|
|
"loss": 0.6083,
|
|
"loss/base_kto": 3.8607468605041504,
|
|
"metrics/advantage_var": 256.79180908203125,
|
|
"regularization/lipschitz_norm": 1023.462890625,
|
|
"regularization/mmd": 0.14622294902801514,
|
|
"regularization/rkhs_norm": 174.07493591308594,
|
|
"regularization/w1": 0.8597087860107422,
|
|
"rewards/chosen": -0.036556180619515516,
|
|
"rewards/margins": 0.13006937143262418,
|
|
"rewards/rejected": -0.1666255520521397,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 23.316652297973633,
|
|
"kl": 8.898043632507324,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36317914.29457364,
|
|
"logits/rejected": -36388060.92598425,
|
|
"logps/chosen": -473.58173449612406,
|
|
"logps/rejected": -385.36141732283465,
|
|
"loss": 0.5983,
|
|
"loss/base_kto": 3.79764986038208,
|
|
"metrics/advantage_var": 227.89488220214844,
|
|
"regularization/lipschitz_norm": 1005.82177734375,
|
|
"regularization/mmd": 0.1439458131790161,
|
|
"regularization/rkhs_norm": 171.3640594482422,
|
|
"regularization/w1": 0.844890296459198,
|
|
"rewards/chosen": 0.13743141824884933,
|
|
"rewards/margins": 0.18320775239566434,
|
|
"rewards/rejected": -0.04577633414681502,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 25.074560165405273,
|
|
"kl": 12.272727012634277,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35246064.807121664,
|
|
"logits/rejected": -37174958.04620462,
|
|
"logps/chosen": -480.1280600890208,
|
|
"logps/rejected": -371.32485045379536,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.7572388648986816,
|
|
"metrics/advantage_var": 257.8333435058594,
|
|
"regularization/lipschitz_norm": 1083.5467529296875,
|
|
"regularization/mmd": 0.15022344887256622,
|
|
"regularization/rkhs_norm": 178.83743286132812,
|
|
"regularization/w1": 0.9101791381835938,
|
|
"rewards/chosen": 0.2493885164798544,
|
|
"rewards/margins": 0.22527819138102984,
|
|
"rewards/rejected": 0.02411032509882458,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 27.07461929321289,
|
|
"kl": 13.4547758102417,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37319737.146304674,
|
|
"logits/rejected": -39135580.525121555,
|
|
"logps/chosen": -490.99589932126696,
|
|
"logps/rejected": -354.61826377633713,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.783406972885132,
|
|
"metrics/advantage_var": 230.5780487060547,
|
|
"regularization/lipschitz_norm": 1008.3624267578125,
|
|
"regularization/mmd": 0.14437010884284973,
|
|
"regularization/rkhs_norm": 171.8691864013672,
|
|
"regularization/w1": 0.8470243811607361,
|
|
"rewards/chosen": 0.2892926476480015,
|
|
"rewards/margins": 0.18616953459387747,
|
|
"rewards/rejected": 0.10312311305412404,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 18.292905807495117,
|
|
"kl": 10.209785461425781,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36234257.794628754,
|
|
"logits/rejected": -37591432.50695518,
|
|
"logps/chosen": -492.9997037914692,
|
|
"logps/rejected": -383.4734833848532,
|
|
"loss": 0.6027,
|
|
"loss/base_kto": 3.817885160446167,
|
|
"metrics/advantage_var": 226.81895446777344,
|
|
"regularization/lipschitz_norm": 1024.1492919921875,
|
|
"regularization/mmd": 0.1432671993970871,
|
|
"regularization/rkhs_norm": 170.55618286132812,
|
|
"regularization/w1": 0.8602854013442993,
|
|
"rewards/chosen": 0.15244419766828346,
|
|
"rewards/margins": 0.16707033068643515,
|
|
"rewards/rejected": -0.014626133018151686,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 23.79297637939453,
|
|
"kl": 12.521636009216309,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37608606.73556231,
|
|
"logits/rejected": -38464521.8778135,
|
|
"logps/chosen": -495.2281534954407,
|
|
"logps/rejected": -359.3269192122186,
|
|
"loss": 0.5985,
|
|
"loss/base_kto": 3.8062989711761475,
|
|
"metrics/advantage_var": 231.9058837890625,
|
|
"regularization/lipschitz_norm": 994.4779663085938,
|
|
"regularization/mmd": 0.14625613391399384,
|
|
"regularization/rkhs_norm": 174.1144561767578,
|
|
"regularization/w1": 0.8353614807128906,
|
|
"rewards/chosen": 0.1706936613042304,
|
|
"rewards/margins": 0.15506315391171507,
|
|
"rewards/rejected": 0.015630507392515324,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 23.27012825012207,
|
|
"kl": 9.972247123718262,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35928514.10989011,
|
|
"logits/rejected": -37505377.54276828,
|
|
"logps/chosen": -476.1001275510204,
|
|
"logps/rejected": -362.91332134525663,
|
|
"loss": 0.5939,
|
|
"loss/base_kto": 3.7839455604553223,
|
|
"metrics/advantage_var": 220.28575134277344,
|
|
"regularization/lipschitz_norm": 983.1730346679688,
|
|
"regularization/mmd": 0.14116238057613373,
|
|
"regularization/rkhs_norm": 168.05044555664062,
|
|
"regularization/w1": 0.8258653879165649,
|
|
"rewards/chosen": 0.11913015703952855,
|
|
"rewards/margins": 0.20158931759065607,
|
|
"rewards/rejected": -0.08245916055112752,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 20.798995971679688,
|
|
"kl": 11.124133110046387,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36712182.92758089,
|
|
"logits/rejected": -37702862.09825674,
|
|
"logps/chosen": -486.5741525423729,
|
|
"logps/rejected": -387.107493066561,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.753119707107544,
|
|
"metrics/advantage_var": 252.92202758789062,
|
|
"regularization/lipschitz_norm": 1070.2249755859375,
|
|
"regularization/mmd": 0.15119387209415436,
|
|
"regularization/rkhs_norm": 179.99270629882812,
|
|
"regularization/w1": 0.8989889025688171,
|
|
"rewards/chosen": 0.17683574047588238,
|
|
"rewards/margins": 0.21209049551113768,
|
|
"rewards/rejected": -0.0352547550352553,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 19.908632278442383,
|
|
"kl": 18.188589096069336,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35877641.68229955,
|
|
"logits/rejected": -39142381.802907914,
|
|
"logps/chosen": -501.7586989409985,
|
|
"logps/rejected": -371.3272667609047,
|
|
"loss": 0.6015,
|
|
"loss/base_kto": 3.7630486488342285,
|
|
"metrics/advantage_var": 293.6002502441406,
|
|
"regularization/lipschitz_norm": 1062.2041015625,
|
|
"regularization/mmd": 0.15667079389095306,
|
|
"regularization/rkhs_norm": 186.5128631591797,
|
|
"regularization/w1": 0.8922514319419861,
|
|
"rewards/chosen": 0.3207323821698303,
|
|
"rewards/margins": 0.19457858106989592,
|
|
"rewards/rejected": 0.12615380109993438,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 19.364850997924805,
|
|
"kl": 15.621047019958496,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36409356.740279935,
|
|
"logits/rejected": -37735952.87912088,
|
|
"logps/chosen": -461.5768856920684,
|
|
"logps/rejected": -362.5079719387755,
|
|
"loss": 0.6017,
|
|
"loss/base_kto": 3.7713749408721924,
|
|
"metrics/advantage_var": 290.8183288574219,
|
|
"regularization/lipschitz_norm": 1055.6641845703125,
|
|
"regularization/mmd": 0.15516722202301025,
|
|
"regularization/rkhs_norm": 184.72288513183594,
|
|
"regularization/w1": 0.8867579698562622,
|
|
"rewards/chosen": 0.17923610799976308,
|
|
"rewards/margins": 0.20985018538965836,
|
|
"rewards/rejected": -0.030614077389895262,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 26.162830352783203,
|
|
"kl": 12.588196754455566,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35913031.8817734,
|
|
"logits/rejected": -37448497.97913562,
|
|
"logps/chosen": -472.5988300492611,
|
|
"logps/rejected": -351.2942203800298,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.7862350940704346,
|
|
"metrics/advantage_var": 238.59634399414062,
|
|
"regularization/lipschitz_norm": 1020.0418090820312,
|
|
"regularization/mmd": 0.14619143307209015,
|
|
"regularization/rkhs_norm": 174.0374298095703,
|
|
"regularization/w1": 0.8568350672721863,
|
|
"rewards/chosen": 0.20795296604801672,
|
|
"rewards/margins": 0.18811387111569933,
|
|
"rewards/rejected": 0.019839094932317378,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 23.330114364624023,
|
|
"kl": 6.336676120758057,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37333538.13333333,
|
|
"logits/rejected": -37922161.96129032,
|
|
"logps/chosen": -509.4089962121212,
|
|
"logps/rejected": -372.30751008064516,
|
|
"loss": 0.5984,
|
|
"loss/base_kto": 3.7602767944335938,
|
|
"metrics/advantage_var": 242.1787872314453,
|
|
"regularization/lipschitz_norm": 1043.499267578125,
|
|
"regularization/mmd": 0.15003834664821625,
|
|
"regularization/rkhs_norm": 178.61708068847656,
|
|
"regularization/w1": 0.876539409160614,
|
|
"rewards/chosen": 0.07642408428770123,
|
|
"rewards/margins": 0.22191090364726523,
|
|
"rewards/rejected": -0.145486819359564,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 23.97354507446289,
|
|
"kl": 5.9447503089904785,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -37274766.36515913,
|
|
"logits/rejected": -37945961.698389456,
|
|
"logps/chosen": -473.22236180904525,
|
|
"logps/rejected": -402.6048682284041,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.755023956298828,
|
|
"metrics/advantage_var": 260.27764892578125,
|
|
"regularization/lipschitz_norm": 954.7799072265625,
|
|
"regularization/mmd": 0.14579203724861145,
|
|
"regularization/rkhs_norm": 173.5619659423828,
|
|
"regularization/w1": 0.8020151257514954,
|
|
"rewards/chosen": -0.05309266580808502,
|
|
"rewards/margins": 0.21499916431007476,
|
|
"rewards/rejected": -0.2680918301181598,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.679683685302734,
|
|
"kl": 7.433067321777344,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36799846.1633282,
|
|
"logits/rejected": -37531141.67987322,
|
|
"logps/chosen": -497.2073382126348,
|
|
"logps/rejected": -372.8678437004754,
|
|
"loss": 0.6039,
|
|
"loss/base_kto": 3.754338026046753,
|
|
"metrics/advantage_var": 282.3930358886719,
|
|
"regularization/lipschitz_norm": 1097.113525390625,
|
|
"regularization/mmd": 0.15551340579986572,
|
|
"regularization/rkhs_norm": 185.135009765625,
|
|
"regularization/w1": 0.9215753674507141,
|
|
"rewards/chosen": 0.04501990213967251,
|
|
"rewards/margins": 0.22417894263606342,
|
|
"rewards/rejected": -0.1791590404963909,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 23.896312713623047,
|
|
"kl": 7.550872325897217,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35131958.25165563,
|
|
"logits/rejected": -36617428.07100592,
|
|
"logps/chosen": -489.2178704470199,
|
|
"logps/rejected": -374.2630131286982,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.7019195556640625,
|
|
"metrics/advantage_var": 252.4306182861328,
|
|
"regularization/lipschitz_norm": 1060.677978515625,
|
|
"regularization/mmd": 0.15066249668598175,
|
|
"regularization/rkhs_norm": 179.360107421875,
|
|
"regularization/w1": 0.890969455242157,
|
|
"rewards/chosen": 0.1339371030693812,
|
|
"rewards/margins": 0.27898328076980766,
|
|
"rewards/rejected": -0.14504617770042647,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 18.10948371887207,
|
|
"kl": 8.011310577392578,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36319811.855421685,
|
|
"logits/rejected": -36256867.74025974,
|
|
"logps/chosen": -509.9324171686747,
|
|
"logps/rejected": -378.48736810064935,
|
|
"loss": 0.6103,
|
|
"loss/base_kto": 3.7063615322113037,
|
|
"metrics/advantage_var": 322.354736328125,
|
|
"regularization/lipschitz_norm": 1196.734375,
|
|
"regularization/mmd": 0.17048956453800201,
|
|
"regularization/rkhs_norm": 202.96377563476562,
|
|
"regularization/w1": 1.0052570104599,
|
|
"rewards/chosen": 0.09192125481295299,
|
|
"rewards/margins": 0.3045674667095769,
|
|
"rewards/rejected": -0.2126462118966239,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 19.786163330078125,
|
|
"kl": 7.736794471740723,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36661231.72019078,
|
|
"logits/rejected": -36563991.59447005,
|
|
"logps/chosen": -497.6756756756757,
|
|
"logps/rejected": -370.278561827957,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.7038872241973877,
|
|
"metrics/advantage_var": 262.55804443359375,
|
|
"regularization/lipschitz_norm": 1069.9774169921875,
|
|
"regularization/mmd": 0.1515924483537674,
|
|
"regularization/rkhs_norm": 180.4672088623047,
|
|
"regularization/w1": 0.8987810015678406,
|
|
"rewards/chosen": 0.07784664839364008,
|
|
"rewards/margins": 0.26766477722263515,
|
|
"rewards/rejected": -0.18981812882899504,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 20.511571884155273,
|
|
"kl": 3.772076368331909,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35221650.985645935,
|
|
"logits/rejected": -36026054.37059724,
|
|
"logps/chosen": -481.38093102073367,
|
|
"logps/rejected": -363.369807618683,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.7333037853240967,
|
|
"metrics/advantage_var": 246.35459899902344,
|
|
"regularization/lipschitz_norm": 1037.7484130859375,
|
|
"regularization/mmd": 0.14947380125522614,
|
|
"regularization/rkhs_norm": 177.94500732421875,
|
|
"regularization/w1": 0.8717086911201477,
|
|
"rewards/chosen": -0.04120655151075153,
|
|
"rewards/margins": 0.2596291908262986,
|
|
"rewards/rejected": -0.3008357423370501,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 26.289173126220703,
|
|
"kl": 6.652445316314697,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -34508889.740219094,
|
|
"logits/rejected": -36907094.265210606,
|
|
"logps/chosen": -473.2691705790297,
|
|
"logps/rejected": -393.4323322932917,
|
|
"loss": 0.5959,
|
|
"loss/base_kto": 3.726672410964966,
|
|
"metrics/advantage_var": 251.7250213623047,
|
|
"regularization/lipschitz_norm": 1058.9185791015625,
|
|
"regularization/mmd": 0.15085165202617645,
|
|
"regularization/rkhs_norm": 179.58531188964844,
|
|
"regularization/w1": 0.8894916772842407,
|
|
"rewards/chosen": 0.026274503489988325,
|
|
"rewards/margins": 0.2647168634500117,
|
|
"rewards/rejected": -0.23844235996002341,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 18.68597984313965,
|
|
"kl": 9.16284465789795,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37543116.8,
|
|
"logits/rejected": -37562058.45496183,
|
|
"logps/chosen": -468.5819,
|
|
"logps/rejected": -398.0042223282443,
|
|
"loss": 0.589,
|
|
"loss/base_kto": 3.7274672985076904,
|
|
"metrics/advantage_var": 227.504150390625,
|
|
"regularization/lipschitz_norm": 1000.2185668945312,
|
|
"regularization/mmd": 0.1447226107120514,
|
|
"regularization/rkhs_norm": 172.28883361816406,
|
|
"regularization/w1": 0.840183675289154,
|
|
"rewards/chosen": 0.08327081298828125,
|
|
"rewards/margins": 0.24724660579302837,
|
|
"rewards/rejected": -0.16397579280474714,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 16.821556091308594,
|
|
"kl": 4.419839859008789,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35375996.88816856,
|
|
"logits/rejected": -36667855.34841629,
|
|
"logps/chosen": -494.91491085899514,
|
|
"logps/rejected": -366.35020739064856,
|
|
"loss": 0.5959,
|
|
"loss/base_kto": 3.7440781593322754,
|
|
"metrics/advantage_var": 236.4251251220703,
|
|
"regularization/lipschitz_norm": 1040.6561279296875,
|
|
"regularization/mmd": 0.1487843245267868,
|
|
"regularization/rkhs_norm": 177.1241912841797,
|
|
"regularization/w1": 0.8741510510444641,
|
|
"rewards/chosen": -0.0041946360125920375,
|
|
"rewards/margins": 0.2443924415467971,
|
|
"rewards/rejected": -0.24858707755938914,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 25.17261505126953,
|
|
"kl": 8.986940383911133,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35114056.02446483,
|
|
"logits/rejected": -36799792.25559106,
|
|
"logps/chosen": -486.995747324159,
|
|
"logps/rejected": -369.1325878594249,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.724412679672241,
|
|
"metrics/advantage_var": 244.765380859375,
|
|
"regularization/lipschitz_norm": 1072.1748046875,
|
|
"regularization/mmd": 0.1519133448600769,
|
|
"regularization/rkhs_norm": 180.84922790527344,
|
|
"regularization/w1": 0.9006267786026001,
|
|
"rewards/chosen": 0.0687982751688826,
|
|
"rewards/margins": 0.2667971264209418,
|
|
"rewards/rejected": -0.1979988512520592,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 22.074447631835938,
|
|
"kl": 6.470108985900879,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36523317.13207547,
|
|
"logits/rejected": -35150660.37267081,
|
|
"logps/chosen": -476.89740566037733,
|
|
"logps/rejected": -384.77763489906835,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.7635300159454346,
|
|
"metrics/advantage_var": 244.6377410888672,
|
|
"regularization/lipschitz_norm": 1048.344970703125,
|
|
"regularization/mmd": 0.14969810843467712,
|
|
"regularization/rkhs_norm": 178.2120361328125,
|
|
"regularization/w1": 0.8806098103523254,
|
|
"rewards/chosen": 0.038192071254898166,
|
|
"rewards/margins": 0.2267126093954261,
|
|
"rewards/rejected": -0.18852053814052794,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 20.145780563354492,
|
|
"kl": 5.831607341766357,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35429127.95031056,
|
|
"logits/rejected": -36183316.93081761,
|
|
"logps/chosen": -489.85141692546586,
|
|
"logps/rejected": -366.0193592767296,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.7904281616210938,
|
|
"metrics/advantage_var": 229.73941040039062,
|
|
"regularization/lipschitz_norm": 1026.3841552734375,
|
|
"regularization/mmd": 0.14527571201324463,
|
|
"regularization/rkhs_norm": 172.94728088378906,
|
|
"regularization/w1": 0.8621627688407898,
|
|
"rewards/chosen": 0.06258985862968872,
|
|
"rewards/margins": 0.2060081287167809,
|
|
"rewards/rejected": -0.14341827008709218,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 25.634119033813477,
|
|
"kl": 11.515710830688477,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36382145.28992248,
|
|
"logits/rejected": -36669056.2015748,
|
|
"logps/chosen": -490.12751937984495,
|
|
"logps/rejected": -366.40905511811025,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.7585716247558594,
|
|
"metrics/advantage_var": 260.51385498046875,
|
|
"regularization/lipschitz_norm": 1048.564208984375,
|
|
"regularization/mmd": 0.15253324806690216,
|
|
"regularization/rkhs_norm": 181.5872039794922,
|
|
"regularization/w1": 0.8807938694953918,
|
|
"rewards/chosen": 0.19663611123728197,
|
|
"rewards/margins": 0.24336114557556476,
|
|
"rewards/rejected": -0.04672503433828279,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 18.166215896606445,
|
|
"kl": 11.50452709197998,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36945996.34359805,
|
|
"logits/rejected": -37733933.56259427,
|
|
"logps/chosen": -480.5982576985413,
|
|
"logps/rejected": -384.8634992458522,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.759248733520508,
|
|
"metrics/advantage_var": 252.5819091796875,
|
|
"regularization/lipschitz_norm": 1028.3375244140625,
|
|
"regularization/mmd": 0.1492318958044052,
|
|
"regularization/rkhs_norm": 177.6570281982422,
|
|
"regularization/w1": 0.863803505897522,
|
|
"rewards/chosen": 0.12525890480177523,
|
|
"rewards/margins": 0.22652064785536685,
|
|
"rewards/rejected": -0.10126174305359163,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 18.386093139648438,
|
|
"kl": 3.3252906799316406,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36452709.43396226,
|
|
"logits/rejected": -36395106.583850935,
|
|
"logps/chosen": -473.59443789308176,
|
|
"logps/rejected": -368.72030279503105,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.7615044116973877,
|
|
"metrics/advantage_var": 228.7191925048828,
|
|
"regularization/lipschitz_norm": 998.3923950195312,
|
|
"regularization/mmd": 0.147730752825737,
|
|
"regularization/rkhs_norm": 175.8699493408203,
|
|
"regularization/w1": 0.838649570941925,
|
|
"rewards/chosen": -0.2154238718860554,
|
|
"rewards/margins": 0.23705703662034985,
|
|
"rewards/rejected": -0.45248090850640527,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 18.495468139648438,
|
|
"kl": 12.432347297668457,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35935427.56090225,
|
|
"logits/rejected": -37345929.36585366,
|
|
"logps/chosen": -505.1909304511278,
|
|
"logps/rejected": -361.523831300813,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.736987352371216,
|
|
"metrics/advantage_var": 243.1018829345703,
|
|
"regularization/lipschitz_norm": 1043.049072265625,
|
|
"regularization/mmd": 0.14960911870002747,
|
|
"regularization/rkhs_norm": 178.1060791015625,
|
|
"regularization/w1": 0.8761612176895142,
|
|
"rewards/chosen": 0.11253662109375,
|
|
"rewards/margins": 0.236124351935658,
|
|
"rewards/rejected": -0.12358773084190802,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 23.46388053894043,
|
|
"kl": 15.903279304504395,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -37170248.916408665,
|
|
"logits/rejected": -38285085.164556965,
|
|
"logps/chosen": -470.14033475232196,
|
|
"logps/rejected": -361.36130340189874,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.545806884765625,
|
|
"metrics/advantage_var": 382.6809387207031,
|
|
"regularization/lipschitz_norm": 1265.8563232421875,
|
|
"regularization/mmd": 0.18023906648159027,
|
|
"regularization/rkhs_norm": 214.5703125,
|
|
"regularization/w1": 1.063319206237793,
|
|
"rewards/chosen": 0.41920735740071113,
|
|
"rewards/margins": 0.4605262327255881,
|
|
"rewards/rejected": -0.041318875324877005,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 15.666631698608398,
|
|
"kl": 8.914244651794434,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36493937.21452145,
|
|
"logits/rejected": -37904912.712166175,
|
|
"logps/chosen": -472.21158209570956,
|
|
"logps/rejected": -366.03354506676556,
|
|
"loss": 0.6057,
|
|
"loss/base_kto": 3.293743133544922,
|
|
"metrics/advantage_var": 567.812744140625,
|
|
"regularization/lipschitz_norm": 1584.333251953125,
|
|
"regularization/mmd": 0.22094635665416718,
|
|
"regularization/rkhs_norm": 263.0313720703125,
|
|
"regularization/w1": 1.330839991569519,
|
|
"rewards/chosen": 0.2899548873649572,
|
|
"rewards/margins": 0.7817628193272392,
|
|
"rewards/rejected": -0.49180793196228206,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 19.461841583251953,
|
|
"kl": 8.61931324005127,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34194705.39020537,
|
|
"logits/rejected": -36160532.57496136,
|
|
"logps/chosen": -465.3089454976303,
|
|
"logps/rejected": -381.4887944358578,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.2844083309173584,
|
|
"metrics/advantage_var": 520.3375244140625,
|
|
"regularization/lipschitz_norm": 1457.4146728515625,
|
|
"regularization/mmd": 0.21181674301624298,
|
|
"regularization/rkhs_norm": 252.1627655029297,
|
|
"regularization/w1": 1.2242282629013062,
|
|
"rewards/chosen": 0.3697112024677873,
|
|
"rewards/margins": 0.7881369142867596,
|
|
"rewards/rejected": -0.4184257118189722,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 17.235340118408203,
|
|
"kl": 8.328340530395508,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36064603.897435896,
|
|
"logits/rejected": -38002900.29268292,
|
|
"logps/chosen": -488.20122195512823,
|
|
"logps/rejected": -379.7233708079268,
|
|
"loss": 0.6142,
|
|
"loss/base_kto": 3.3217384815216064,
|
|
"metrics/advantage_var": 621.2462158203125,
|
|
"regularization/lipschitz_norm": 1624.1302490234375,
|
|
"regularization/mmd": 0.22741980850696564,
|
|
"regularization/rkhs_norm": 270.7378845214844,
|
|
"regularization/w1": 1.364269495010376,
|
|
"rewards/chosen": 0.29768960903852415,
|
|
"rewards/margins": 0.7642958422166396,
|
|
"rewards/rejected": -0.46660623317811545,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 17.388891220092773,
|
|
"kl": 10.026690483093262,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35639451.79080824,
|
|
"logits/rejected": -37012058.72419106,
|
|
"logps/chosen": -473.23028922345486,
|
|
"logps/rejected": -375.4877937211094,
|
|
"loss": 0.6192,
|
|
"loss/base_kto": 3.289081573486328,
|
|
"metrics/advantage_var": 648.875732421875,
|
|
"regularization/lipschitz_norm": 1700.989501953125,
|
|
"regularization/mmd": 0.23589801788330078,
|
|
"regularization/rkhs_norm": 280.83099365234375,
|
|
"regularization/w1": 1.4288312196731567,
|
|
"rewards/chosen": 0.36237759597705776,
|
|
"rewards/margins": 0.7931990426588519,
|
|
"rewards/rejected": -0.4308214466817941,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 17.387683868408203,
|
|
"kl": 9.051863670349121,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34377488.20253164,
|
|
"logits/rejected": -35814782.41975309,
|
|
"logps/chosen": -458.64912974683546,
|
|
"logps/rejected": -369.98336226851853,
|
|
"loss": 0.6127,
|
|
"loss/base_kto": 3.3329391479492188,
|
|
"metrics/advantage_var": 592.2706298828125,
|
|
"regularization/lipschitz_norm": 1600.7916259765625,
|
|
"regularization/mmd": 0.2243695706129074,
|
|
"regularization/rkhs_norm": 267.10662841796875,
|
|
"regularization/w1": 1.3446649312973022,
|
|
"rewards/chosen": 0.29426552977743026,
|
|
"rewards/margins": 0.7542412080211851,
|
|
"rewards/rejected": -0.45997567824375485,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 20.747802734375,
|
|
"kl": 25.399429321289062,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35270650.13180516,
|
|
"logits/rejected": -34957487.94501718,
|
|
"logps/chosen": -461.7970988538682,
|
|
"logps/rejected": -372.5379886168385,
|
|
"loss": 0.6098,
|
|
"loss/base_kto": 3.24271821975708,
|
|
"metrics/advantage_var": 652.7489624023438,
|
|
"regularization/lipschitz_norm": 1659.84765625,
|
|
"regularization/mmd": 0.2411045879125595,
|
|
"regularization/rkhs_norm": 287.0292663574219,
|
|
"regularization/w1": 1.394271969795227,
|
|
"rewards/chosen": 0.710110552331438,
|
|
"rewards/margins": 0.7775980461545124,
|
|
"rewards/rejected": -0.0674874938230744,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 17.625804901123047,
|
|
"kl": 23.202255249023438,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35154289.097288676,
|
|
"logits/rejected": -36876360.13476264,
|
|
"logps/chosen": -468.1279405901116,
|
|
"logps/rejected": -350.28072358346094,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.2979683876037598,
|
|
"metrics/advantage_var": 507.5339050292969,
|
|
"regularization/lipschitz_norm": 1512.8592529296875,
|
|
"regularization/mmd": 0.2121281623840332,
|
|
"regularization/rkhs_norm": 252.53355407714844,
|
|
"regularization/w1": 1.2708017826080322,
|
|
"rewards/chosen": 0.6044021436092005,
|
|
"rewards/margins": 0.7162313083566403,
|
|
"rewards/rejected": -0.1118291647474397,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 22.623109817504883,
|
|
"kl": 19.243173599243164,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35866076.57861635,
|
|
"logits/rejected": -37102938.63354037,
|
|
"logps/chosen": -473.47553066037733,
|
|
"logps/rejected": -367.97522806677017,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.318005323410034,
|
|
"metrics/advantage_var": 539.412109375,
|
|
"regularization/lipschitz_norm": 1509.506103515625,
|
|
"regularization/mmd": 0.21407151222229004,
|
|
"regularization/rkhs_norm": 254.84703063964844,
|
|
"regularization/w1": 1.267985224723816,
|
|
"rewards/chosen": 0.5582143915524272,
|
|
"rewards/margins": 0.7233219527468057,
|
|
"rewards/rejected": -0.1651075611943784,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 16.42879867553711,
|
|
"kl": 18.079944610595703,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35420267.622874804,
|
|
"logits/rejected": -36134869.93996841,
|
|
"logps/chosen": -488.09254250386397,
|
|
"logps/rejected": -349.1705667456556,
|
|
"loss": 0.6009,
|
|
"loss/base_kto": 3.287992238998413,
|
|
"metrics/advantage_var": 551.7247924804688,
|
|
"regularization/lipschitz_norm": 1549.2371826171875,
|
|
"regularization/mmd": 0.21788766980171204,
|
|
"regularization/rkhs_norm": 259.39007568359375,
|
|
"regularization/w1": 1.3013591766357422,
|
|
"rewards/chosen": 0.5241970905380603,
|
|
"rewards/margins": 0.7846226762441474,
|
|
"rewards/rejected": -0.2604255857060871,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 23.41855812072754,
|
|
"kl": 15.331746101379395,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35697496.65967017,
|
|
"logits/rejected": -35984195.2365416,
|
|
"logps/chosen": -497.9211488005997,
|
|
"logps/rejected": -373.3093902936378,
|
|
"loss": 0.6132,
|
|
"loss/base_kto": 3.2378342151641846,
|
|
"metrics/advantage_var": 707.39111328125,
|
|
"regularization/lipschitz_norm": 1703.13671875,
|
|
"regularization/mmd": 0.23753051459789276,
|
|
"regularization/rkhs_norm": 282.7744140625,
|
|
"regularization/w1": 1.4306347370147705,
|
|
"rewards/chosen": 0.5982461497522723,
|
|
"rewards/margins": 0.87108940330645,
|
|
"rewards/rejected": -0.2728432535541777,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 27.812116622924805,
|
|
"kl": 11.39678955078125,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35754067.65506808,
|
|
"logits/rejected": -37962892.613893375,
|
|
"logps/chosen": -500.95243948562785,
|
|
"logps/rejected": -376.2573202746365,
|
|
"loss": 0.6097,
|
|
"loss/base_kto": 3.3006317615509033,
|
|
"metrics/advantage_var": 606.1578979492188,
|
|
"regularization/lipschitz_norm": 1606.619384765625,
|
|
"regularization/mmd": 0.2278004139661789,
|
|
"regularization/rkhs_norm": 271.19097900390625,
|
|
"regularization/w1": 1.349560260772705,
|
|
"rewards/chosen": 0.45513879080583397,
|
|
"rewards/margins": 0.7843247080330906,
|
|
"rewards/rejected": -0.32918591722725665,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 19.89821434020996,
|
|
"kl": 17.57761573791504,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35095525.65916399,
|
|
"logits/rejected": -36683713.75075988,
|
|
"logps/chosen": -457.8347065916399,
|
|
"logps/rejected": -377.5421732522796,
|
|
"loss": 0.6111,
|
|
"loss/base_kto": 3.3101792335510254,
|
|
"metrics/advantage_var": 605.47900390625,
|
|
"regularization/lipschitz_norm": 1612.7193603515625,
|
|
"regularization/mmd": 0.22387544810771942,
|
|
"regularization/rkhs_norm": 266.51837158203125,
|
|
"regularization/w1": 1.3546842336654663,
|
|
"rewards/chosen": 0.4465755941016881,
|
|
"rewards/margins": 0.7688103072995296,
|
|
"rewards/rejected": -0.32223471319784147,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 16.23716163635254,
|
|
"kl": 15.947044372558594,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35381671.27503975,
|
|
"logits/rejected": -36699244.534562215,
|
|
"logps/chosen": -490.22908386327504,
|
|
"logps/rejected": -382.8509024577573,
|
|
"loss": 0.6055,
|
|
"loss/base_kto": 3.2874481678009033,
|
|
"metrics/advantage_var": 624.5247192382812,
|
|
"regularization/lipschitz_norm": 1582.3753662109375,
|
|
"regularization/mmd": 0.22732551395893097,
|
|
"regularization/rkhs_norm": 270.6255798339844,
|
|
"regularization/w1": 1.329195261001587,
|
|
"rewards/chosen": 0.48758283897120924,
|
|
"rewards/margins": 0.7722326424961348,
|
|
"rewards/rejected": -0.2846498035249256,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 17.053136825561523,
|
|
"kl": 7.763891696929932,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -33607524.4556962,
|
|
"logits/rejected": -36076942.222222224,
|
|
"logps/chosen": -479.6942246835443,
|
|
"logps/rejected": -397.61176215277777,
|
|
"loss": 0.6094,
|
|
"loss/base_kto": 3.3501861095428467,
|
|
"metrics/advantage_var": 555.3399658203125,
|
|
"regularization/lipschitz_norm": 1555.4041748046875,
|
|
"regularization/mmd": 0.21865926682949066,
|
|
"regularization/rkhs_norm": 260.30865478515625,
|
|
"regularization/w1": 1.3065394163131714,
|
|
"rewards/chosen": 0.2237238823613034,
|
|
"rewards/margins": 0.7349968308265329,
|
|
"rewards/rejected": -0.5112729484652295,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 14.961041450500488,
|
|
"kl": 11.09844970703125,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35691696.551724136,
|
|
"logits/rejected": -36306205.65089723,
|
|
"logps/chosen": -483.1750374812594,
|
|
"logps/rejected": -375.51238784665577,
|
|
"loss": 0.6081,
|
|
"loss/base_kto": 3.3126275539398193,
|
|
"metrics/advantage_var": 605.9591674804688,
|
|
"regularization/lipschitz_norm": 1581.5926513671875,
|
|
"regularization/mmd": 0.22356970608234406,
|
|
"regularization/rkhs_norm": 266.1544494628906,
|
|
"regularization/w1": 1.3285378217697144,
|
|
"rewards/chosen": 0.4672240295867691,
|
|
"rewards/margins": 0.762672867987182,
|
|
"rewards/rejected": -0.2954488384004129,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 17.15966796875,
|
|
"kl": 21.29660415649414,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36913877.91251885,
|
|
"logits/rejected": -37147354.243111834,
|
|
"logps/chosen": -463.0637254901961,
|
|
"logps/rejected": -381.3438512965964,
|
|
"loss": 0.6052,
|
|
"loss/base_kto": 3.240344285964966,
|
|
"metrics/advantage_var": 625.4959716796875,
|
|
"regularization/lipschitz_norm": 1630.3070068359375,
|
|
"regularization/mmd": 0.23164783418178558,
|
|
"regularization/rkhs_norm": 275.7712097167969,
|
|
"regularization/w1": 1.3694578409194946,
|
|
"rewards/chosen": 0.658963072533701,
|
|
"rewards/margins": 0.800113734107312,
|
|
"rewards/rejected": -0.14115066157361097,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 19.81757354736328,
|
|
"kl": 12.617999076843262,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35385363.1700468,
|
|
"logits/rejected": -36406318.47261346,
|
|
"logps/chosen": -505.6136895475819,
|
|
"logps/rejected": -390.85529147104853,
|
|
"loss": 0.6248,
|
|
"loss/base_kto": 3.2752506732940674,
|
|
"metrics/advantage_var": 878.1770629882812,
|
|
"regularization/lipschitz_norm": 1757.767822265625,
|
|
"regularization/mmd": 0.24669337272644043,
|
|
"regularization/rkhs_norm": 293.6826171875,
|
|
"regularization/w1": 1.476525068283081,
|
|
"rewards/chosen": 0.4399619734789392,
|
|
"rewards/margins": 0.8107024550510978,
|
|
"rewards/rejected": -0.37074048157215866,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.518264770507812,
|
|
"kl": 14.232144355773926,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34325926.380517505,
|
|
"logits/rejected": -35629148.04494382,
|
|
"logps/chosen": -510.90934170471843,
|
|
"logps/rejected": -366.26991372391655,
|
|
"loss": 0.6156,
|
|
"loss/base_kto": 3.2792961597442627,
|
|
"metrics/advantage_var": 641.0203247070312,
|
|
"regularization/lipschitz_norm": 1685.8836669921875,
|
|
"regularization/mmd": 0.22928786277770996,
|
|
"regularization/rkhs_norm": 272.9617614746094,
|
|
"regularization/w1": 1.416142225265503,
|
|
"rewards/chosen": 0.5655688513722412,
|
|
"rewards/margins": 0.79264980730623,
|
|
"rewards/rejected": -0.22708095593398878,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 15.789749145507812,
|
|
"kl": 13.984519004821777,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -33883500.26259542,
|
|
"logits/rejected": -36367466.496,
|
|
"logps/chosen": -458.6433683206107,
|
|
"logps/rejected": -372.42095,
|
|
"loss": 0.6085,
|
|
"loss/base_kto": 3.382080554962158,
|
|
"metrics/advantage_var": 484.20098876953125,
|
|
"regularization/lipschitz_norm": 1517.2821044921875,
|
|
"regularization/mmd": 0.2113133668899536,
|
|
"regularization/rkhs_norm": 251.5635223388672,
|
|
"regularization/w1": 1.2745169401168823,
|
|
"rewards/chosen": 0.4072445469048187,
|
|
"rewards/margins": 0.6396539219048187,
|
|
"rewards/rejected": -0.232409375,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 21.53681755065918,
|
|
"kl": 15.689457893371582,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34532404.59214502,
|
|
"logits/rejected": -36046218.35598706,
|
|
"logps/chosen": -482.3880287009063,
|
|
"logps/rejected": -389.77806432038835,
|
|
"loss": 0.6181,
|
|
"loss/base_kto": 3.359943151473999,
|
|
"metrics/advantage_var": 712.8927001953125,
|
|
"regularization/lipschitz_norm": 1613.7772216796875,
|
|
"regularization/mmd": 0.22953668236732483,
|
|
"regularization/rkhs_norm": 273.2579650878906,
|
|
"regularization/w1": 1.3555727005004883,
|
|
"rewards/chosen": 0.4487912273118863,
|
|
"rewards/margins": 0.6958919586276225,
|
|
"rewards/rejected": -0.24710073131573623,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 22.96624755859375,
|
|
"kl": 14.630607604980469,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35050179.7888,
|
|
"logits/rejected": -35679105.36793893,
|
|
"logps/chosen": -466.9254,
|
|
"logps/rejected": -386.56760496183205,
|
|
"loss": 0.6128,
|
|
"loss/base_kto": 3.312497854232788,
|
|
"metrics/advantage_var": 719.9235229492188,
|
|
"regularization/lipschitz_norm": 1617.9178466796875,
|
|
"regularization/mmd": 0.2305762767791748,
|
|
"regularization/rkhs_norm": 274.49554443359375,
|
|
"regularization/w1": 1.359050989151001,
|
|
"rewards/chosen": 0.44150859375,
|
|
"rewards/margins": 0.7217417760019085,
|
|
"rewards/rejected": -0.2802331822519084,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 15.745792388916016,
|
|
"kl": 8.091479301452637,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -32896367.97398374,
|
|
"logits/rejected": -35968195.56090225,
|
|
"logps/chosen": -502.90264227642274,
|
|
"logps/rejected": -401.7211466165414,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.322103261947632,
|
|
"metrics/advantage_var": 561.0201416015625,
|
|
"regularization/lipschitz_norm": 1558.4539794921875,
|
|
"regularization/mmd": 0.2188023328781128,
|
|
"regularization/rkhs_norm": 260.4789733886719,
|
|
"regularization/w1": 1.3091013431549072,
|
|
"rewards/chosen": 0.2893491915571011,
|
|
"rewards/margins": 0.7215985160401839,
|
|
"rewards/rejected": -0.43224932448308273,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 19.984336853027344,
|
|
"kl": 7.116430759429932,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33699011.56090225,
|
|
"logits/rejected": -34270282.92682927,
|
|
"logps/chosen": -492.8671522556391,
|
|
"logps/rejected": -374.3812245934959,
|
|
"loss": 0.6038,
|
|
"loss/base_kto": 3.3200013637542725,
|
|
"metrics/advantage_var": 527.3442993164062,
|
|
"regularization/lipschitz_norm": 1544.6370849609375,
|
|
"regularization/mmd": 0.21295630931854248,
|
|
"regularization/rkhs_norm": 253.5194091796875,
|
|
"regularization/w1": 1.2974952459335327,
|
|
"rewards/chosen": 0.37293168835173873,
|
|
"rewards/margins": 0.755537512395819,
|
|
"rewards/rejected": -0.3826058240440803,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.519479751586914,
|
|
"kl": 19.29461097717285,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35132220.87559055,
|
|
"logits/rejected": -36775164.427906975,
|
|
"logps/chosen": -492.6152559055118,
|
|
"logps/rejected": -384.59292635658915,
|
|
"loss": 0.611,
|
|
"loss/base_kto": 3.2485482692718506,
|
|
"metrics/advantage_var": 672.3517456054688,
|
|
"regularization/lipschitz_norm": 1671.7054443359375,
|
|
"regularization/mmd": 0.2353748083114624,
|
|
"regularization/rkhs_norm": 280.2080993652344,
|
|
"regularization/w1": 1.404232382774353,
|
|
"rewards/chosen": 0.6023383193128691,
|
|
"rewards/margins": 0.8477697668964447,
|
|
"rewards/rejected": -0.24543144758357557,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 20.922687530517578,
|
|
"kl": 15.558287620544434,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35004855.08116386,
|
|
"logits/rejected": -37826664.523126,
|
|
"logps/chosen": -517.2947932618683,
|
|
"logps/rejected": -353.1667663476874,
|
|
"loss": 0.6014,
|
|
"loss/base_kto": 3.2946345806121826,
|
|
"metrics/advantage_var": 542.4263916015625,
|
|
"regularization/lipschitz_norm": 1545.110107421875,
|
|
"regularization/mmd": 0.21872659027576447,
|
|
"regularization/rkhs_norm": 260.3888244628906,
|
|
"regularization/w1": 1.297892451286316,
|
|
"rewards/chosen": 0.5169487057929747,
|
|
"rewards/margins": 0.7500582727568155,
|
|
"rewards/rejected": -0.23310956696384072,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 22.38857650756836,
|
|
"kl": 14.668024063110352,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34744646.492753625,
|
|
"logits/rejected": -36934725.92412747,
|
|
"logps/chosen": -473.3769122383253,
|
|
"logps/rejected": -366.36186456752654,
|
|
"loss": 0.5927,
|
|
"loss/base_kto": 3.2817132472991943,
|
|
"metrics/advantage_var": 545.3973999023438,
|
|
"regularization/lipschitz_norm": 1478.9656982421875,
|
|
"regularization/mmd": 0.21779386699199677,
|
|
"regularization/rkhs_norm": 259.2784118652344,
|
|
"regularization/w1": 1.2423312664031982,
|
|
"rewards/chosen": 0.500670944435009,
|
|
"rewards/margins": 0.7776237545343546,
|
|
"rewards/rejected": -0.2769528100993456,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 14.221234321594238,
|
|
"kl": 10.854016304016113,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34245188.58566978,
|
|
"logits/rejected": -34834194.45768025,
|
|
"logps/chosen": -484.12816394080994,
|
|
"logps/rejected": -383.6113832288401,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.323260545730591,
|
|
"metrics/advantage_var": 517.9158325195312,
|
|
"regularization/lipschitz_norm": 1449.843994140625,
|
|
"regularization/mmd": 0.21208129823207855,
|
|
"regularization/rkhs_norm": 252.4777374267578,
|
|
"regularization/w1": 1.2178689241409302,
|
|
"rewards/chosen": 0.3794784308222595,
|
|
"rewards/margins": 0.7277580669302752,
|
|
"rewards/rejected": -0.34827963610801577,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 16.409099578857422,
|
|
"kl": 8.480694770812988,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33930395.35399674,
|
|
"logits/rejected": -36168929.679160416,
|
|
"logps/chosen": -471.6565048939641,
|
|
"logps/rejected": -368.127202023988,
|
|
"loss": 0.6056,
|
|
"loss/base_kto": 3.3039956092834473,
|
|
"metrics/advantage_var": 585.809814453125,
|
|
"regularization/lipschitz_norm": 1569.984375,
|
|
"regularization/mmd": 0.22230716049671173,
|
|
"regularization/rkhs_norm": 264.6513977050781,
|
|
"regularization/w1": 1.318786859512329,
|
|
"rewards/chosen": 0.396398472747056,
|
|
"rewards/margins": 0.7698608351937624,
|
|
"rewards/rejected": -0.37346236244670633,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.707660675048828,
|
|
"kl": 5.820353984832764,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33180988.281345565,
|
|
"logits/rejected": -35673778.300319485,
|
|
"logps/chosen": -474.0996750764526,
|
|
"logps/rejected": -364.0633236821086,
|
|
"loss": 0.611,
|
|
"loss/base_kto": 3.3307483196258545,
|
|
"metrics/advantage_var": 571.0430908203125,
|
|
"regularization/lipschitz_norm": 1590.1129150390625,
|
|
"regularization/mmd": 0.22141209244728088,
|
|
"regularization/rkhs_norm": 263.5858459472656,
|
|
"regularization/w1": 1.3356949090957642,
|
|
"rewards/chosen": 0.3691172001923261,
|
|
"rewards/margins": 0.763298253670823,
|
|
"rewards/rejected": -0.3941810534784969,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 19.33246612548828,
|
|
"kl": 12.9971923828125,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35619480.03852327,
|
|
"logits/rejected": -35304982.599695586,
|
|
"logps/chosen": -501.06887038523274,
|
|
"logps/rejected": -395.90353881278537,
|
|
"loss": 0.6055,
|
|
"loss/base_kto": 3.255815267562866,
|
|
"metrics/advantage_var": 651.4505004882812,
|
|
"regularization/lipschitz_norm": 1618.5975341796875,
|
|
"regularization/mmd": 0.22837963700294495,
|
|
"regularization/rkhs_norm": 271.8804931640625,
|
|
"regularization/w1": 1.3596218824386597,
|
|
"rewards/chosen": 0.45599404422276285,
|
|
"rewards/margins": 0.7864263960884877,
|
|
"rewards/rejected": -0.3304323518657249,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.47831916809082,
|
|
"kl": 7.972372531890869,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33950480.0,
|
|
"logits/rejected": -36250624.0,
|
|
"logps/chosen": -472.679833984375,
|
|
"logps/rejected": -379.3304931640625,
|
|
"loss": 0.6203,
|
|
"loss/base_kto": 3.3692805767059326,
|
|
"metrics/advantage_var": 684.947998046875,
|
|
"regularization/lipschitz_norm": 1621.791015625,
|
|
"regularization/mmd": 0.23118801414966583,
|
|
"regularization/rkhs_norm": 275.2238464355469,
|
|
"regularization/w1": 1.362304449081421,
|
|
"rewards/chosen": 0.3005537509918213,
|
|
"rewards/margins": 0.7356410503387452,
|
|
"rewards/rejected": -0.4350872993469238,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 19.20001792907715,
|
|
"kl": 8.95646858215332,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -35575203.11844078,
|
|
"logits/rejected": -36071951.86949429,
|
|
"logps/chosen": -470.80312968515744,
|
|
"logps/rejected": -359.3844820554649,
|
|
"loss": 0.6108,
|
|
"loss/base_kto": 3.354182481765747,
|
|
"metrics/advantage_var": 972.58935546875,
|
|
"regularization/lipschitz_norm": 1557.8624267578125,
|
|
"regularization/mmd": 0.22396443784236908,
|
|
"regularization/rkhs_norm": 266.62432861328125,
|
|
"regularization/w1": 1.3086044788360596,
|
|
"rewards/chosen": 0.4345947906173866,
|
|
"rewards/margins": 0.6453848304764302,
|
|
"rewards/rejected": -0.21079003985904363,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 20.125614166259766,
|
|
"kl": 13.368906021118164,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34601246.52464229,
|
|
"logits/rejected": -36016706.85099846,
|
|
"logps/chosen": -497.65759141494436,
|
|
"logps/rejected": -386.39669738863284,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.247060775756836,
|
|
"metrics/advantage_var": 507.5834045410156,
|
|
"regularization/lipschitz_norm": 1515.463134765625,
|
|
"regularization/mmd": 0.21229799091815948,
|
|
"regularization/rkhs_norm": 252.7357177734375,
|
|
"regularization/w1": 1.2729891538619995,
|
|
"rewards/chosen": 0.47443781672297297,
|
|
"rewards/margins": 0.7902570604045687,
|
|
"rewards/rejected": -0.31581924368159564,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.01663589477539,
|
|
"kl": 11.805732727050781,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35699617.74726989,
|
|
"logits/rejected": -36404623.02347418,
|
|
"logps/chosen": -479.7743272230889,
|
|
"logps/rejected": -362.94219483568077,
|
|
"loss": 0.6157,
|
|
"loss/base_kto": 3.3279221057891846,
|
|
"metrics/advantage_var": 676.853515625,
|
|
"regularization/lipschitz_norm": 1625.8232421875,
|
|
"regularization/mmd": 0.2319633960723877,
|
|
"regularization/rkhs_norm": 276.1469421386719,
|
|
"regularization/w1": 1.3656915426254272,
|
|
"rewards/chosen": 0.4625465047898791,
|
|
"rewards/margins": 0.7707515974334138,
|
|
"rewards/rejected": -0.3082050926435348,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 13.947898864746094,
|
|
"kl": 13.337854385375977,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34399215.69426752,
|
|
"logits/rejected": -36257270.57668712,
|
|
"logps/chosen": -465.81847133757964,
|
|
"logps/rejected": -391.2774635736196,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.2883949279785156,
|
|
"metrics/advantage_var": 553.4547729492188,
|
|
"regularization/lipschitz_norm": 1512.5606689453125,
|
|
"regularization/mmd": 0.21815328299999237,
|
|
"regularization/rkhs_norm": 259.706298828125,
|
|
"regularization/w1": 1.2705509662628174,
|
|
"rewards/chosen": 0.4106421015065187,
|
|
"rewards/margins": 0.76340555245672,
|
|
"rewards/rejected": -0.3527634509502013,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 18.118799209594727,
|
|
"kl": 9.493661880493164,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34988798.801872075,
|
|
"logits/rejected": -37884916.78247261,
|
|
"logps/chosen": -488.5215483619345,
|
|
"logps/rejected": -390.90145735524254,
|
|
"loss": 0.607,
|
|
"loss/base_kto": 3.3376197814941406,
|
|
"metrics/advantage_var": 566.1810913085938,
|
|
"regularization/lipschitz_norm": 1550.6263427734375,
|
|
"regularization/mmd": 0.21609444916248322,
|
|
"regularization/rkhs_norm": 257.2553405761719,
|
|
"regularization/w1": 1.3025261163711548,
|
|
"rewards/chosen": 0.40235855650046315,
|
|
"rewards/margins": 0.7242311781352195,
|
|
"rewards/rejected": -0.32187262163475644,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 16.39398193359375,
|
|
"kl": 12.16468334197998,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33297530.24496124,
|
|
"logits/rejected": -35110537.877165355,
|
|
"logps/chosen": -505.63289728682173,
|
|
"logps/rejected": -365.0585137795276,
|
|
"loss": 0.5919,
|
|
"loss/base_kto": 3.3040432929992676,
|
|
"metrics/advantage_var": 535.5570678710938,
|
|
"regularization/lipschitz_norm": 1454.2027587890625,
|
|
"regularization/mmd": 0.2095969170331955,
|
|
"regularization/rkhs_norm": 249.52017211914062,
|
|
"regularization/w1": 1.2215303182601929,
|
|
"rewards/chosen": 0.43538458772407945,
|
|
"rewards/margins": 0.747115736992042,
|
|
"rewards/rejected": -0.3117311492679626,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 15.860579490661621,
|
|
"kl": 17.849750518798828,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33140769.684210528,
|
|
"logits/rejected": -34944121.9047619,
|
|
"logps/chosen": -491.50308388157896,
|
|
"logps/rejected": -368.12323288690476,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.363266706466675,
|
|
"metrics/advantage_var": 462.5398864746094,
|
|
"regularization/lipschitz_norm": 1432.591064453125,
|
|
"regularization/mmd": 0.20235076546669006,
|
|
"regularization/rkhs_norm": 240.89378356933594,
|
|
"regularization/w1": 1.2033764123916626,
|
|
"rewards/chosen": 0.4306080466822574,
|
|
"rewards/margins": 0.6535257850972034,
|
|
"rewards/rejected": -0.22291773841494605,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 18.264732360839844,
|
|
"kl": 7.774613857269287,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35857124.00633915,
|
|
"logits/rejected": -36097744.12364761,
|
|
"logps/chosen": -500.95963748019017,
|
|
"logps/rejected": -373.39608288253476,
|
|
"loss": 0.5693,
|
|
"loss/base_kto": 3.2240264415740967,
|
|
"metrics/advantage_var": 439.531494140625,
|
|
"regularization/lipschitz_norm": 1351.1187744140625,
|
|
"regularization/mmd": 0.1954774707555771,
|
|
"regularization/rkhs_norm": 232.71128845214844,
|
|
"regularization/w1": 1.1349399089813232,
|
|
"rewards/chosen": 0.34736526900350384,
|
|
"rewards/margins": 0.8281013661948678,
|
|
"rewards/rejected": -0.480736097191364,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 18.854707717895508,
|
|
"kl": 10.137467384338379,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34253770.784251966,
|
|
"logits/rejected": -35558099.94418605,
|
|
"logps/chosen": -483.59124015748034,
|
|
"logps/rejected": -391.21017441860465,
|
|
"loss": 0.5646,
|
|
"loss/base_kto": 3.1491477489471436,
|
|
"metrics/advantage_var": 486.41265869140625,
|
|
"regularization/lipschitz_norm": 1383.4696044921875,
|
|
"regularization/mmd": 0.20591209828853607,
|
|
"regularization/rkhs_norm": 245.1334228515625,
|
|
"regularization/w1": 1.1621143817901611,
|
|
"rewards/chosen": 0.44636917714997537,
|
|
"rewards/margins": 0.9409108835604647,
|
|
"rewards/rejected": -0.49454170641048933,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 18.3592472076416,
|
|
"kl": 9.977899551391602,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34074335.41818182,
|
|
"logits/rejected": -34795017.909677416,
|
|
"logps/chosen": -480.08267045454545,
|
|
"logps/rejected": -382.16597782258066,
|
|
"loss": 0.5656,
|
|
"loss/base_kto": 3.227848768234253,
|
|
"metrics/advantage_var": 430.7770690917969,
|
|
"regularization/lipschitz_norm": 1311.7689208984375,
|
|
"regularization/mmd": 0.19471898674964905,
|
|
"regularization/rkhs_norm": 231.8083038330078,
|
|
"regularization/w1": 1.1018857955932617,
|
|
"rewards/chosen": 0.4298485495827415,
|
|
"rewards/margins": 0.8052986838600853,
|
|
"rewards/rejected": -0.37545013427734375,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 15.637130737304688,
|
|
"kl": 14.094367027282715,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34749183.17684887,
|
|
"logits/rejected": -36952633.58054711,
|
|
"logps/chosen": -480.34324758842445,
|
|
"logps/rejected": -406.4927811550152,
|
|
"loss": 0.5578,
|
|
"loss/base_kto": 3.229459524154663,
|
|
"metrics/advantage_var": 405.9147033691406,
|
|
"regularization/lipschitz_norm": 1242.7464599609375,
|
|
"regularization/mmd": 0.18878209590911865,
|
|
"regularization/rkhs_norm": 224.74058532714844,
|
|
"regularization/w1": 1.0439070463180542,
|
|
"rewards/chosen": 0.45734741603446544,
|
|
"rewards/margins": 0.7996827778746178,
|
|
"rewards/rejected": -0.34233536184015245,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 13.556568145751953,
|
|
"kl": 14.232521057128906,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33980316.644757435,
|
|
"logits/rejected": -35438048.84867395,
|
|
"logps/chosen": -454.9765258215962,
|
|
"logps/rejected": -376.20151618564745,
|
|
"loss": 0.5577,
|
|
"loss/base_kto": 3.192952871322632,
|
|
"metrics/advantage_var": 395.028076171875,
|
|
"regularization/lipschitz_norm": 1284.7835693359375,
|
|
"regularization/mmd": 0.1895955353975296,
|
|
"regularization/rkhs_norm": 225.708984375,
|
|
"regularization/w1": 1.0792182683944702,
|
|
"rewards/chosen": 0.5560843358763693,
|
|
"rewards/margins": 0.8412919087300648,
|
|
"rewards/rejected": -0.2852075728536954,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 13.461142539978027,
|
|
"kl": 10.573742866516113,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35907306.409638554,
|
|
"logits/rejected": -36226772.77922078,
|
|
"logps/chosen": -476.1554028614458,
|
|
"logps/rejected": -376.61071935876623,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.193457841873169,
|
|
"metrics/advantage_var": 422.6647033691406,
|
|
"regularization/lipschitz_norm": 1323.8089599609375,
|
|
"regularization/mmd": 0.19564370810985565,
|
|
"regularization/rkhs_norm": 232.9091796875,
|
|
"regularization/w1": 1.1119993925094604,
|
|
"rewards/chosen": 0.5121253140001412,
|
|
"rewards/margins": 0.8491723707184473,
|
|
"rewards/rejected": -0.3370470567183061,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.828086853027344,
|
|
"kl": 15.678369522094727,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34839754.25194401,
|
|
"logits/rejected": -36932558.16640502,
|
|
"logps/chosen": -498.4690902021773,
|
|
"logps/rejected": -363.9652178178964,
|
|
"loss": 0.5588,
|
|
"loss/base_kto": 3.134873151779175,
|
|
"metrics/advantage_var": 448.7715759277344,
|
|
"regularization/lipschitz_norm": 1352.0111083984375,
|
|
"regularization/mmd": 0.1995236575603485,
|
|
"regularization/rkhs_norm": 237.52818298339844,
|
|
"regularization/w1": 1.1356892585754395,
|
|
"rewards/chosen": 0.5871664613809778,
|
|
"rewards/margins": 0.899272894812223,
|
|
"rewards/rejected": -0.3121064334312451,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.24053382873535,
|
|
"kl": 15.413047790527344,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35107295.79874214,
|
|
"logits/rejected": -36519191.850931674,
|
|
"logps/chosen": -477.8340212264151,
|
|
"logps/rejected": -370.28091032608694,
|
|
"loss": 0.5574,
|
|
"loss/base_kto": 3.1444289684295654,
|
|
"metrics/advantage_var": 446.54595947265625,
|
|
"regularization/lipschitz_norm": 1329.8575439453125,
|
|
"regularization/mmd": 0.19787751138210297,
|
|
"regularization/rkhs_norm": 235.5684814453125,
|
|
"regularization/w1": 1.1170803308486938,
|
|
"rewards/chosen": 0.6172305413012235,
|
|
"rewards/margins": 0.8998891360368844,
|
|
"rewards/rejected": -0.2826585947356609,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 18.77298927307129,
|
|
"kl": 15.185454368591309,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34475537.87936508,
|
|
"logits/rejected": -34764655.064615384,
|
|
"logps/chosen": -466.21299603174606,
|
|
"logps/rejected": -382.4767548076923,
|
|
"loss": 0.5639,
|
|
"loss/base_kto": 3.199817419052124,
|
|
"metrics/advantage_var": 464.99267578125,
|
|
"regularization/lipschitz_norm": 1322.8828125,
|
|
"regularization/mmd": 0.2003309726715088,
|
|
"regularization/rkhs_norm": 238.48928833007812,
|
|
"regularization/w1": 1.1112215518951416,
|
|
"rewards/chosen": 0.5393206883990576,
|
|
"rewards/margins": 0.8461695841322306,
|
|
"rewards/rejected": -0.3068488957331731,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 13.673155784606934,
|
|
"kl": 13.778063774108887,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35899871.179487176,
|
|
"logits/rejected": -37279079.02439024,
|
|
"logps/chosen": -476.42618189102564,
|
|
"logps/rejected": -368.0221512957317,
|
|
"loss": 0.5601,
|
|
"loss/base_kto": 3.1171321868896484,
|
|
"metrics/advantage_var": 463.0811462402344,
|
|
"regularization/lipschitz_norm": 1378.9393310546875,
|
|
"regularization/mmd": 0.20512448251247406,
|
|
"regularization/rkhs_norm": 244.19580078125,
|
|
"regularization/w1": 1.158308982849121,
|
|
"rewards/chosen": 0.5845967806302584,
|
|
"rewards/margins": 0.9282775113103985,
|
|
"rewards/rejected": -0.34368073068014005,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 23.332332611083984,
|
|
"kl": 8.331280708312988,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34493173.94561934,
|
|
"logits/rejected": -35933833.52750809,
|
|
"logps/chosen": -488.2673716012085,
|
|
"logps/rejected": -369.0847491909385,
|
|
"loss": 0.5674,
|
|
"loss/base_kto": 3.2024307250976562,
|
|
"metrics/advantage_var": 443.2652282714844,
|
|
"regularization/lipschitz_norm": 1356.251953125,
|
|
"regularization/mmd": 0.19784317910671234,
|
|
"regularization/rkhs_norm": 235.527587890625,
|
|
"regularization/w1": 1.139251708984375,
|
|
"rewards/chosen": 0.44392067716200906,
|
|
"rewards/margins": 0.8574803284596678,
|
|
"rewards/rejected": -0.41355965129765876,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 12.615058898925781,
|
|
"kl": 14.106646537780762,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34501215.293375395,
|
|
"logits/rejected": -35575801.65944272,
|
|
"logps/chosen": -486.3713525236593,
|
|
"logps/rejected": -389.772663506192,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.1935501098632812,
|
|
"metrics/advantage_var": 429.70709228515625,
|
|
"regularization/lipschitz_norm": 1347.1920166015625,
|
|
"regularization/mmd": 0.19866220653057098,
|
|
"regularization/rkhs_norm": 236.5026397705078,
|
|
"regularization/w1": 1.1316413879394531,
|
|
"rewards/chosen": 0.551907897371599,
|
|
"rewards/margins": 0.8530093457377463,
|
|
"rewards/rejected": -0.30110144836614744,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 11.8766450881958,
|
|
"kl": 9.767699241638184,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34407625.030674845,
|
|
"logits/rejected": -35954462.98089172,
|
|
"logps/chosen": -466.3296587423313,
|
|
"logps/rejected": -371.3240694665605,
|
|
"loss": 0.5562,
|
|
"loss/base_kto": 3.204167127609253,
|
|
"metrics/advantage_var": 416.6337890625,
|
|
"regularization/lipschitz_norm": 1260.052490234375,
|
|
"regularization/mmd": 0.18705010414123535,
|
|
"regularization/rkhs_norm": 222.6787109375,
|
|
"regularization/w1": 1.0584440231323242,
|
|
"rewards/chosen": 0.4848421248921587,
|
|
"rewards/margins": 0.8548203067675941,
|
|
"rewards/rejected": -0.3699781818754354,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 19.986351013183594,
|
|
"kl": 12.02132511138916,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35264085.2053973,
|
|
"logits/rejected": -36831589.4812398,
|
|
"logps/chosen": -525.3279610194902,
|
|
"logps/rejected": -358.79674755301795,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.1256330013275146,
|
|
"metrics/advantage_var": 464.7733459472656,
|
|
"regularization/lipschitz_norm": 1385.0789794921875,
|
|
"regularization/mmd": 0.20345798134803772,
|
|
"regularization/rkhs_norm": 242.2118682861328,
|
|
"regularization/w1": 1.1634663343429565,
|
|
"rewards/chosen": 0.5855761389324868,
|
|
"rewards/margins": 0.9408044572138006,
|
|
"rewards/rejected": -0.35522831828131374,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 16.817529678344727,
|
|
"kl": 13.309791564941406,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34030450.26438569,
|
|
"logits/rejected": -35974666.448979594,
|
|
"logps/chosen": -491.5963258164852,
|
|
"logps/rejected": -366.8942062401884,
|
|
"loss": 0.5662,
|
|
"loss/base_kto": 3.195519208908081,
|
|
"metrics/advantage_var": 431.6280822753906,
|
|
"regularization/lipschitz_norm": 1354.5709228515625,
|
|
"regularization/mmd": 0.19629018008708954,
|
|
"regularization/rkhs_norm": 233.6787872314453,
|
|
"regularization/w1": 1.137839674949646,
|
|
"rewards/chosen": 0.5386746467364891,
|
|
"rewards/margins": 0.8309032616274575,
|
|
"rewards/rejected": -0.2922286148909684,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 14.734240531921387,
|
|
"kl": 12.235140800476074,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34690800.89314195,
|
|
"logits/rejected": -35920201.3108729,
|
|
"logps/chosen": -451.43072169059013,
|
|
"logps/rejected": -386.42381316998467,
|
|
"loss": 0.562,
|
|
"loss/base_kto": 3.219853162765503,
|
|
"metrics/advantage_var": 410.29229736328125,
|
|
"regularization/lipschitz_norm": 1291.866455078125,
|
|
"regularization/mmd": 0.19108930230140686,
|
|
"regularization/rkhs_norm": 227.4872589111328,
|
|
"regularization/w1": 1.0851677656173706,
|
|
"rewards/chosen": 0.46488648434384966,
|
|
"rewards/margins": 0.7992691300954289,
|
|
"rewards/rejected": -0.3343826457515793,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 16.336462020874023,
|
|
"kl": 11.372636795043945,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33382796.996923078,
|
|
"logits/rejected": -35981312.0,
|
|
"logps/chosen": -476.9077884615385,
|
|
"logps/rejected": -370.9947668650794,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.2600109577178955,
|
|
"metrics/advantage_var": 375.123291015625,
|
|
"regularization/lipschitz_norm": 1227.2027587890625,
|
|
"regularization/mmd": 0.18363702297210693,
|
|
"regularization/rkhs_norm": 218.6155242919922,
|
|
"regularization/w1": 1.0308502912521362,
|
|
"rewards/chosen": 0.4624712665264423,
|
|
"rewards/margins": 0.773748068710532,
|
|
"rewards/rejected": -0.31127680218408976,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 16.161535263061523,
|
|
"kl": 14.80101490020752,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34102726.7368421,
|
|
"logits/rejected": -36646887.61904762,
|
|
"logps/chosen": -475.85166529605266,
|
|
"logps/rejected": -376.9000651041667,
|
|
"loss": 0.5656,
|
|
"loss/base_kto": 3.2036683559417725,
|
|
"metrics/advantage_var": 450.4566955566406,
|
|
"regularization/lipschitz_norm": 1339.655517578125,
|
|
"regularization/mmd": 0.1957130879163742,
|
|
"regularization/rkhs_norm": 232.9917449951172,
|
|
"regularization/w1": 1.1253107786178589,
|
|
"rewards/chosen": 0.4932795072856702,
|
|
"rewards/margins": 0.8310514679528717,
|
|
"rewards/rejected": -0.33777196066720144,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 14.53366470336914,
|
|
"kl": 10.798033714294434,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34299629.88307692,
|
|
"logits/rejected": -36437073.26984127,
|
|
"logps/chosen": -479.0939903846154,
|
|
"logps/rejected": -361.65558035714287,
|
|
"loss": 0.5561,
|
|
"loss/base_kto": 3.213702917098999,
|
|
"metrics/advantage_var": 419.8805847167969,
|
|
"regularization/lipschitz_norm": 1243.016845703125,
|
|
"regularization/mmd": 0.19072774052619934,
|
|
"regularization/rkhs_norm": 227.0568084716797,
|
|
"regularization/w1": 1.0441340208053589,
|
|
"rewards/chosen": 0.4662467604417067,
|
|
"rewards/margins": 0.8341530705080509,
|
|
"rewards/rejected": -0.3679063100663442,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 28.190868377685547,
|
|
"kl": 9.8163480758667,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -33925865.020472445,
|
|
"logits/rejected": -35565123.47286822,
|
|
"logps/chosen": -509.15536417322835,
|
|
"logps/rejected": -383.68989825581394,
|
|
"loss": 0.5645,
|
|
"loss/base_kto": 3.1935627460479736,
|
|
"metrics/advantage_var": 452.0596618652344,
|
|
"regularization/lipschitz_norm": 1339.458251953125,
|
|
"regularization/mmd": 0.19692611694335938,
|
|
"regularization/rkhs_norm": 234.4358367919922,
|
|
"regularization/w1": 1.1251448392868042,
|
|
"rewards/chosen": 0.5196519746555118,
|
|
"rewards/margins": 0.8694590751733171,
|
|
"rewards/rejected": -0.3498071005178052,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 12.313821792602539,
|
|
"kl": 10.951972961425781,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34793547.3664,
|
|
"logits/rejected": -36536376.28091603,
|
|
"logps/chosen": -461.0857,
|
|
"logps/rejected": -379.19544370229005,
|
|
"loss": 0.5673,
|
|
"loss/base_kto": 3.2096824645996094,
|
|
"metrics/advantage_var": 425.0608825683594,
|
|
"regularization/lipschitz_norm": 1349.4190673828125,
|
|
"regularization/mmd": 0.19493000209331512,
|
|
"regularization/rkhs_norm": 232.05955505371094,
|
|
"regularization/w1": 1.133512020111084,
|
|
"rewards/chosen": 0.489045751953125,
|
|
"rewards/margins": 0.8332719193553196,
|
|
"rewards/rejected": -0.34422616740219464,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 19.709569931030273,
|
|
"kl": 11.492325782775879,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35572068.03692307,
|
|
"logits/rejected": -36022005.43492063,
|
|
"logps/chosen": -484.4163461538462,
|
|
"logps/rejected": -394.89290674603177,
|
|
"loss": 0.5579,
|
|
"loss/base_kto": 3.1914687156677246,
|
|
"metrics/advantage_var": 415.55322265625,
|
|
"regularization/lipschitz_norm": 1284.950927734375,
|
|
"regularization/mmd": 0.19253580272197723,
|
|
"regularization/rkhs_norm": 229.2092742919922,
|
|
"regularization/w1": 1.0793588161468506,
|
|
"rewards/chosen": 0.5305240572415866,
|
|
"rewards/margins": 0.8528915528238039,
|
|
"rewards/rejected": -0.3223674955822173,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 16.271976470947266,
|
|
"kl": 14.540762901306152,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34673287.97377049,
|
|
"logits/rejected": -36120985.6,
|
|
"logps/chosen": -474.97602459016395,
|
|
"logps/rejected": -402.01767723880596,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.16609787940979,
|
|
"metrics/advantage_var": 454.21783447265625,
|
|
"regularization/lipschitz_norm": 1273.1346435546875,
|
|
"regularization/mmd": 0.1932131052017212,
|
|
"regularization/rkhs_norm": 230.01560974121094,
|
|
"regularization/w1": 1.0694330930709839,
|
|
"rewards/chosen": 0.5350604448162142,
|
|
"rewards/margins": 0.8801651519932198,
|
|
"rewards/rejected": -0.3451047071770056,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 19.394756317138672,
|
|
"kl": 9.60827350616455,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34917468.79491256,
|
|
"logits/rejected": -35498811.37941628,
|
|
"logps/chosen": -508.1017984896661,
|
|
"logps/rejected": -360.7724174347158,
|
|
"loss": 0.5578,
|
|
"loss/base_kto": 3.186643362045288,
|
|
"metrics/advantage_var": 428.6121520996094,
|
|
"regularization/lipschitz_norm": 1291.1190185546875,
|
|
"regularization/mmd": 0.1908429116010666,
|
|
"regularization/rkhs_norm": 227.19395446777344,
|
|
"regularization/w1": 1.0845400094985962,
|
|
"rewards/chosen": 0.4465728420142339,
|
|
"rewards/margins": 0.8603376198350144,
|
|
"rewards/rejected": -0.4137647778207805,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 13.327056884765625,
|
|
"kl": 10.262199401855469,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35023059.2,
|
|
"logits/rejected": -35387065.6,
|
|
"logps/chosen": -499.890283203125,
|
|
"logps/rejected": -408.4691162109375,
|
|
"loss": 0.558,
|
|
"loss/base_kto": 3.157562255859375,
|
|
"metrics/advantage_var": 443.0968933105469,
|
|
"regularization/lipschitz_norm": 1320.2757568359375,
|
|
"regularization/mmd": 0.19734196364879608,
|
|
"regularization/rkhs_norm": 234.930908203125,
|
|
"regularization/w1": 1.1090315580368042,
|
|
"rewards/chosen": 0.4372579097747803,
|
|
"rewards/margins": 0.8990058898925781,
|
|
"rewards/rejected": -0.46174798011779783,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 19.239852905273438,
|
|
"kl": 9.56927490234375,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34931779.2,
|
|
"logits/rejected": -36211833.6,
|
|
"logps/chosen": -499.37333984375,
|
|
"logps/rejected": -377.584521484375,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.2285377979278564,
|
|
"metrics/advantage_var": 377.6658630371094,
|
|
"regularization/lipschitz_norm": 1293.5233154296875,
|
|
"regularization/mmd": 0.18453995883464813,
|
|
"regularization/rkhs_norm": 219.6904296875,
|
|
"regularization/w1": 1.086559534072876,
|
|
"rewards/chosen": 0.4326962471008301,
|
|
"rewards/margins": 0.796537208557129,
|
|
"rewards/rejected": -0.36384096145629885,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 14.930299758911133,
|
|
"kl": 8.400935173034668,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -33920780.40528634,
|
|
"logits/rejected": -36282470.57095159,
|
|
"logps/chosen": -504.6150881057269,
|
|
"logps/rejected": -352.302535475793,
|
|
"loss": 0.5708,
|
|
"loss/base_kto": 3.201364278793335,
|
|
"metrics/advantage_var": 801.5885620117188,
|
|
"regularization/lipschitz_norm": 1381.3016357421875,
|
|
"regularization/mmd": 0.20444898307323456,
|
|
"regularization/rkhs_norm": 243.3916778564453,
|
|
"regularization/w1": 1.160293459892273,
|
|
"rewards/chosen": 0.4663790396131608,
|
|
"rewards/margins": 0.8207380502363348,
|
|
"rewards/rejected": -0.35435901062317404,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 13.229378700256348,
|
|
"kl": 11.790120124816895,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -32949512.883280758,
|
|
"logits/rejected": -36136027.9380805,
|
|
"logps/chosen": -475.56871056782336,
|
|
"logps/rejected": -353.016060371517,
|
|
"loss": 0.558,
|
|
"loss/base_kto": 3.2167770862579346,
|
|
"metrics/advantage_var": 397.7450256347656,
|
|
"regularization/lipschitz_norm": 1261.3599853515625,
|
|
"regularization/mmd": 0.18782556056976318,
|
|
"regularization/rkhs_norm": 223.6018524169922,
|
|
"regularization/w1": 1.0595422983169556,
|
|
"rewards/chosen": 0.45055083747165814,
|
|
"rewards/margins": 0.8109490739146729,
|
|
"rewards/rejected": -0.3603982364430147,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 18.787633895874023,
|
|
"kl": 11.50531005859375,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34259394.10989011,
|
|
"logits/rejected": -37049517.58631415,
|
|
"logps/chosen": -492.86548273155415,
|
|
"logps/rejected": -375.8316971228616,
|
|
"loss": 0.5632,
|
|
"loss/base_kto": 3.188129425048828,
|
|
"metrics/advantage_var": 464.97637939453125,
|
|
"regularization/lipschitz_norm": 1330.6572265625,
|
|
"regularization/mmd": 0.20006529986858368,
|
|
"regularization/rkhs_norm": 238.1729736328125,
|
|
"regularization/w1": 1.1177520751953125,
|
|
"rewards/chosen": 0.534646553746198,
|
|
"rewards/margins": 0.8828416312181313,
|
|
"rewards/rejected": -0.34819507747193335,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 16.551551818847656,
|
|
"kl": 11.084259033203125,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35763348.08615384,
|
|
"logits/rejected": -37316237.40952381,
|
|
"logps/chosen": -486.15177884615383,
|
|
"logps/rejected": -381.6411706349206,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.210106611251831,
|
|
"metrics/advantage_var": 389.7070007324219,
|
|
"regularization/lipschitz_norm": 1256.907470703125,
|
|
"regularization/mmd": 0.18716497719287872,
|
|
"regularization/rkhs_norm": 222.81546020507812,
|
|
"regularization/w1": 1.0558022260665894,
|
|
"rewards/chosen": 0.4493678166316106,
|
|
"rewards/margins": 0.8265638954619057,
|
|
"rewards/rejected": -0.37719607883029516,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 16.98550033569336,
|
|
"kl": 12.216307640075684,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34792318.37974683,
|
|
"logits/rejected": -35077338.074074075,
|
|
"logps/chosen": -483.32357594936707,
|
|
"logps/rejected": -354.0249324845679,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.1553752422332764,
|
|
"metrics/advantage_var": 476.40966796875,
|
|
"regularization/lipschitz_norm": 1333.2667236328125,
|
|
"regularization/mmd": 0.20372819900512695,
|
|
"regularization/rkhs_norm": 242.53355407714844,
|
|
"regularization/w1": 1.1199439764022827,
|
|
"rewards/chosen": 0.5121294818347013,
|
|
"rewards/margins": 0.9175916762068077,
|
|
"rewards/rejected": -0.40546219437210645,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 12.690099716186523,
|
|
"kl": 11.929011344909668,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35717881.43589743,
|
|
"logits/rejected": -35233954.34146342,
|
|
"logps/chosen": -460.9818709935897,
|
|
"logps/rejected": -404.6694455030488,
|
|
"loss": 0.5577,
|
|
"loss/base_kto": 3.182636022567749,
|
|
"metrics/advantage_var": 408.284423828125,
|
|
"regularization/lipschitz_norm": 1295.5045166015625,
|
|
"regularization/mmd": 0.19084985554218292,
|
|
"regularization/rkhs_norm": 227.2021942138672,
|
|
"regularization/w1": 1.0882236957550049,
|
|
"rewards/chosen": 0.4427344493376903,
|
|
"rewards/margins": 0.8549390522072358,
|
|
"rewards/rejected": -0.4122046028695455,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 22.389938354492188,
|
|
"kl": 10.047457695007324,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34046496.851330206,
|
|
"logits/rejected": -35179457.697347894,
|
|
"logps/chosen": -465.48816510172145,
|
|
"logps/rejected": -375.71504485179406,
|
|
"loss": 0.5529,
|
|
"loss/base_kto": 3.180927038192749,
|
|
"metrics/advantage_var": 409.9590759277344,
|
|
"regularization/lipschitz_norm": 1254.7119140625,
|
|
"regularization/mmd": 0.1885291039943695,
|
|
"regularization/rkhs_norm": 224.4394073486328,
|
|
"regularization/w1": 1.0539579391479492,
|
|
"rewards/chosen": 0.4683889469630282,
|
|
"rewards/margins": 0.8549723912578313,
|
|
"rewards/rejected": -0.38658344429480307,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 16.395126342773438,
|
|
"kl": 10.394868850708008,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35716732.02484472,
|
|
"logits/rejected": -36476116.52830189,
|
|
"logps/chosen": -497.7844526397516,
|
|
"logps/rejected": -370.30829893867923,
|
|
"loss": 0.5642,
|
|
"loss/base_kto": 3.257833480834961,
|
|
"metrics/advantage_var": 408.9557189941406,
|
|
"regularization/lipschitz_norm": 1269.4150390625,
|
|
"regularization/mmd": 0.18956489861011505,
|
|
"regularization/rkhs_norm": 225.67251586914062,
|
|
"regularization/w1": 1.066308617591858,
|
|
"rewards/chosen": 0.4159940162800854,
|
|
"rewards/margins": 0.781420229766475,
|
|
"rewards/rejected": -0.36542621348638954,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 11.82385540008545,
|
|
"kl": 12.206686973571777,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34943426.81761006,
|
|
"logits/rejected": -36785616.298136644,
|
|
"logps/chosen": -471.6402810534591,
|
|
"logps/rejected": -372.4858307453416,
|
|
"loss": 0.5579,
|
|
"loss/base_kto": 3.2004692554473877,
|
|
"metrics/advantage_var": 416.4024353027344,
|
|
"regularization/lipschitz_norm": 1274.0220947265625,
|
|
"regularization/mmd": 0.19225876033306122,
|
|
"regularization/rkhs_norm": 228.87950134277344,
|
|
"regularization/w1": 1.0701783895492554,
|
|
"rewards/chosen": 0.4587482956220519,
|
|
"rewards/margins": 0.8371093756258732,
|
|
"rewards/rejected": -0.37836108000382135,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 22.204914093017578,
|
|
"kl": 9.545755386352539,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35399136.780487806,
|
|
"logits/rejected": -35634733.94871795,
|
|
"logps/chosen": -470.5637385670732,
|
|
"logps/rejected": -389.9996995192308,
|
|
"loss": 0.5681,
|
|
"loss/base_kto": 3.2567946910858154,
|
|
"metrics/advantage_var": 430.03271484375,
|
|
"regularization/lipschitz_norm": 1305.274658203125,
|
|
"regularization/mmd": 0.19192980229854584,
|
|
"regularization/rkhs_norm": 228.48789978027344,
|
|
"regularization/w1": 1.0964306592941284,
|
|
"rewards/chosen": 0.44192607228348896,
|
|
"rewards/margins": 0.8058728891435901,
|
|
"rewards/rejected": -0.36394681686010116,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 14.16252613067627,
|
|
"kl": 9.917055130004883,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34829915.39968404,
|
|
"logits/rejected": -36914095.282843895,
|
|
"logps/chosen": -478.93592022116906,
|
|
"logps/rejected": -363.02965610510046,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.2191874980926514,
|
|
"metrics/advantage_var": 368.8214416503906,
|
|
"regularization/lipschitz_norm": 1223.7933349609375,
|
|
"regularization/mmd": 0.18198032677173615,
|
|
"regularization/rkhs_norm": 216.6432647705078,
|
|
"regularization/w1": 1.0279864072799683,
|
|
"rewards/chosen": 0.4566128265236226,
|
|
"rewards/margins": 0.8056587819398465,
|
|
"rewards/rejected": -0.3490459554162239,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 17.859251022338867,
|
|
"kl": 10.774524688720703,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33632352.150234744,
|
|
"logits/rejected": -35580137.235569425,
|
|
"logps/chosen": -476.5024941314554,
|
|
"logps/rejected": -351.73483814352574,
|
|
"loss": 0.5579,
|
|
"loss/base_kto": 3.259432315826416,
|
|
"metrics/advantage_var": 384.2206726074219,
|
|
"regularization/lipschitz_norm": 1217.2144775390625,
|
|
"regularization/mmd": 0.18147411942481995,
|
|
"regularization/rkhs_norm": 216.04063415527344,
|
|
"regularization/w1": 1.0224602222442627,
|
|
"rewards/chosen": 0.3982738799332453,
|
|
"rewards/margins": 0.7611843961097517,
|
|
"rewards/rejected": -0.36291051617650644,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_loss": 0.5884088716358719,
|
|
"train_runtime": 11134.7372,
|
|
"train_samples_per_second": 13.311,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|