076698cd12
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed2024 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 26.822582244873047,
|
|
"kl": 6.688536167144775,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35990294.6835443,
|
|
"logits/rejected": -37641911.30864198,
|
|
"logps/chosen": -502.6991693037975,
|
|
"logps/rejected": -375.73408564814815,
|
|
"loss": 0.6253,
|
|
"loss/base_kto": 3.941157579421997,
|
|
"metrics/advantage_var": 271.9526062011719,
|
|
"regularization/lipschitz_norm": 1082.0919189453125,
|
|
"regularization/mmd": 0.15222804248332977,
|
|
"regularization/rkhs_norm": 181.22386169433594,
|
|
"regularization/w1": 0.9089571833610535,
|
|
"rewards/chosen": -0.04996775373627868,
|
|
"rewards/margins": 0.049426915031203,
|
|
"rewards/rejected": -0.09939466876748168,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 18.23899269104004,
|
|
"kl": 22.54376792907715,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35605139.52542373,
|
|
"logits/rejected": -37668834.789223455,
|
|
"logps/chosen": -490.5431432973806,
|
|
"logps/rejected": -358.90397187004754,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.8734192848205566,
|
|
"metrics/advantage_var": 181.184326171875,
|
|
"regularization/lipschitz_norm": 904.0665893554688,
|
|
"regularization/mmd": 0.13191047310829163,
|
|
"regularization/rkhs_norm": 157.03627014160156,
|
|
"regularization/w1": 0.7594159841537476,
|
|
"rewards/chosen": 0.40319951179398594,
|
|
"rewards/margins": 0.0966510682581797,
|
|
"rewards/rejected": 0.30654844353580624,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 26.432907104492188,
|
|
"kl": 9.919577598571777,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34675935.179487176,
|
|
"logits/rejected": -35587090.73170732,
|
|
"logps/chosen": -469.19861778846155,
|
|
"logps/rejected": -362.9036775914634,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.903366804122925,
|
|
"metrics/advantage_var": 164.3740997314453,
|
|
"regularization/lipschitz_norm": 849.7183837890625,
|
|
"regularization/mmd": 0.11974674463272095,
|
|
"regularization/rkhs_norm": 142.55564880371094,
|
|
"regularization/w1": 0.713763415813446,
|
|
"rewards/chosen": 0.14293929858085436,
|
|
"rewards/margins": 0.06006812900211603,
|
|
"rewards/rejected": 0.08287116957873833,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 20.65061378479004,
|
|
"kl": 11.507179260253906,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37237064.97007874,
|
|
"logits/rejected": -37540843.36124031,
|
|
"logps/chosen": -505.36899606299215,
|
|
"logps/rejected": -384.1623062015504,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.886489152908325,
|
|
"metrics/advantage_var": 178.0410614013672,
|
|
"regularization/lipschitz_norm": 915.6586303710938,
|
|
"regularization/mmd": 0.1268511265516281,
|
|
"regularization/rkhs_norm": 151.0132598876953,
|
|
"regularization/w1": 0.7691532373428345,
|
|
"rewards/chosen": 0.1602300688976378,
|
|
"rewards/margins": 0.1150228663706877,
|
|
"rewards/rejected": 0.0452072025269501,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 19.4213924407959,
|
|
"kl": 6.736691951751709,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36571694.260336906,
|
|
"logits/rejected": -37923727.31100479,
|
|
"logps/chosen": -490.65974349157733,
|
|
"logps/rejected": -373.0764553429027,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.8735275268554688,
|
|
"metrics/advantage_var": 213.0327911376953,
|
|
"regularization/lipschitz_norm": 984.1552734375,
|
|
"regularization/mmd": 0.1367308646440506,
|
|
"regularization/rkhs_norm": 162.77484130859375,
|
|
"regularization/w1": 0.8266903758049011,
|
|
"rewards/chosen": 0.05937138742911323,
|
|
"rewards/margins": 0.13850803054224498,
|
|
"rewards/rejected": -0.07913664311313173,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 22.295644760131836,
|
|
"kl": 14.757791519165039,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -37726292.421374045,
|
|
"logits/rejected": -37582910.2592,
|
|
"logps/chosen": -491.5485687022901,
|
|
"logps/rejected": -360.84155,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.8581607341766357,
|
|
"metrics/advantage_var": 203.2979736328125,
|
|
"regularization/lipschitz_norm": 937.5336303710938,
|
|
"regularization/mmd": 0.1350373476743698,
|
|
"regularization/rkhs_norm": 160.7587432861328,
|
|
"regularization/w1": 0.7875282168388367,
|
|
"rewards/chosen": 0.24400709312380725,
|
|
"rewards/margins": 0.12322369468630726,
|
|
"rewards/rejected": 0.1207833984375,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 21.928726196289062,
|
|
"kl": 13.538284301757812,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35640385.96319018,
|
|
"logits/rejected": -38540506.49681529,
|
|
"logps/chosen": -485.2036042944785,
|
|
"logps/rejected": -383.4859673566879,
|
|
"loss": 0.5947,
|
|
"loss/base_kto": 3.8243279457092285,
|
|
"metrics/advantage_var": 187.7855682373047,
|
|
"regularization/lipschitz_norm": 953.6726684570312,
|
|
"regularization/mmd": 0.13212864100933075,
|
|
"regularization/rkhs_norm": 157.29600524902344,
|
|
"regularization/w1": 0.8010851144790649,
|
|
"rewards/chosen": 0.2269869611307156,
|
|
"rewards/margins": 0.15800791042760615,
|
|
"rewards/rejected": 0.06897905070310945,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 20.098386764526367,
|
|
"kl": 20.5661678314209,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -37250196.91754123,
|
|
"logits/rejected": -36808780.84176183,
|
|
"logps/chosen": -494.4382496251874,
|
|
"logps/rejected": -372.70360929853183,
|
|
"loss": 0.5873,
|
|
"loss/base_kto": 3.7914528846740723,
|
|
"metrics/advantage_var": 204.13919067382812,
|
|
"regularization/lipschitz_norm": 918.5911254882812,
|
|
"regularization/mmd": 0.13557849824428558,
|
|
"regularization/rkhs_norm": 161.40296936035156,
|
|
"regularization/w1": 0.7716165781021118,
|
|
"rewards/chosen": 0.3903130069486741,
|
|
"rewards/margins": 0.16678777762376076,
|
|
"rewards/rejected": 0.22352522932491334,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 29.430130004882812,
|
|
"kl": 24.841371536254883,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35907025.45454545,
|
|
"logits/rejected": -37089340.532019705,
|
|
"logps/chosen": -465.20985469448584,
|
|
"logps/rejected": -357.6175595238095,
|
|
"loss": 0.5938,
|
|
"loss/base_kto": 3.787205457687378,
|
|
"metrics/advantage_var": 213.7475128173828,
|
|
"regularization/lipschitz_norm": 978.7572631835938,
|
|
"regularization/mmd": 0.1411084234714508,
|
|
"regularization/rkhs_norm": 167.9862060546875,
|
|
"regularization/w1": 0.8221561312675476,
|
|
"rewards/chosen": 0.45094488061487054,
|
|
"rewards/margins": 0.18808537953568077,
|
|
"rewards/rejected": 0.26285950107918976,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 17.58774757385254,
|
|
"kl": 28.271289825439453,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37944043.48961424,
|
|
"logits/rejected": -39779676.09240924,
|
|
"logps/chosen": -486.9328171364985,
|
|
"logps/rejected": -380.546875,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.806340456008911,
|
|
"metrics/advantage_var": 222.38003540039062,
|
|
"regularization/lipschitz_norm": 955.1566772460938,
|
|
"regularization/mmd": 0.14101631939411163,
|
|
"regularization/rkhs_norm": 167.87657165527344,
|
|
"regularization/w1": 0.8023315668106079,
|
|
"rewards/chosen": 0.43112114699728765,
|
|
"rewards/margins": 0.15921002949393576,
|
|
"rewards/rejected": 0.2719111175033519,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 20.11609649658203,
|
|
"kl": 17.88020896911621,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37807972.586846545,
|
|
"logits/rejected": -38566248.710334785,
|
|
"logps/chosen": -478.32419898819563,
|
|
"logps/rejected": -379.9218522561863,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.7347412109375,
|
|
"metrics/advantage_var": 216.2038116455078,
|
|
"regularization/lipschitz_norm": 1012.9171752929688,
|
|
"regularization/mmd": 0.14182019233703613,
|
|
"regularization/rkhs_norm": 168.8335723876953,
|
|
"regularization/w1": 0.850850522518158,
|
|
"rewards/chosen": 0.24950945437658095,
|
|
"rewards/margins": 0.237535000276973,
|
|
"rewards/rejected": 0.011974454099607953,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 17.779966354370117,
|
|
"kl": 8.485206604003906,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36586348.32329635,
|
|
"logits/rejected": -36844611.84591679,
|
|
"logps/chosen": -465.98350832012676,
|
|
"logps/rejected": -377.4280383281972,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.7884018421173096,
|
|
"metrics/advantage_var": 250.99400329589844,
|
|
"regularization/lipschitz_norm": 1028.31787109375,
|
|
"regularization/mmd": 0.14561444520950317,
|
|
"regularization/rkhs_norm": 173.35052490234375,
|
|
"regularization/w1": 0.8637871146202087,
|
|
"rewards/chosen": 0.0761179311906479,
|
|
"rewards/margins": 0.21856869585112315,
|
|
"rewards/rejected": -0.14245076466047524,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 22.252639770507812,
|
|
"kl": 10.02391529083252,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36825978.01869159,
|
|
"logits/rejected": -36777714.35736677,
|
|
"logps/chosen": -478.9975175233645,
|
|
"logps/rejected": -378.5579447492163,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.78965163230896,
|
|
"metrics/advantage_var": 300.36932373046875,
|
|
"regularization/lipschitz_norm": 1015.0419921875,
|
|
"regularization/mmd": 0.14683066308498383,
|
|
"regularization/rkhs_norm": 174.7983856201172,
|
|
"regularization/w1": 0.8526352047920227,
|
|
"rewards/chosen": 0.1416795558275835,
|
|
"rewards/margins": 0.18282197167847097,
|
|
"rewards/rejected": -0.04114241585088748,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 15.103630065917969,
|
|
"kl": 9.661877632141113,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -35989077.19547658,
|
|
"logits/rejected": -38817792.0,
|
|
"logps/chosen": -504.87838247172857,
|
|
"logps/rejected": -364.2305692133132,
|
|
"loss": 0.5882,
|
|
"loss/base_kto": 3.754521608352661,
|
|
"metrics/advantage_var": 207.8542938232422,
|
|
"regularization/lipschitz_norm": 967.5964965820312,
|
|
"regularization/mmd": 0.13858331739902496,
|
|
"regularization/rkhs_norm": 164.9801483154297,
|
|
"regularization/w1": 0.8127809762954712,
|
|
"rewards/chosen": 0.16231197917981371,
|
|
"rewards/margins": 0.21434921681174568,
|
|
"rewards/rejected": -0.052037237631931965,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 27.652835845947266,
|
|
"kl": 5.51276159286499,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36078726.8982036,
|
|
"logits/rejected": -38148705.04575163,
|
|
"logps/chosen": -463.51300523952096,
|
|
"logps/rejected": -374.59369893790847,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.810443162918091,
|
|
"metrics/advantage_var": 228.3494110107422,
|
|
"regularization/lipschitz_norm": 980.9369506835938,
|
|
"regularization/mmd": 0.14567025005817413,
|
|
"regularization/rkhs_norm": 173.41697692871094,
|
|
"regularization/w1": 0.8239870071411133,
|
|
"rewards/chosen": 0.030363008647621748,
|
|
"rewards/margins": 0.18544664625916052,
|
|
"rewards/rejected": -0.15508363761153876,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 24.82827377319336,
|
|
"kl": 14.267743110656738,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37787016.06462035,
|
|
"logits/rejected": -38199646.11195159,
|
|
"logps/chosen": -523.109147819063,
|
|
"logps/rejected": -394.60088880484113,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.7817025184631348,
|
|
"metrics/advantage_var": 229.16064453125,
|
|
"regularization/lipschitz_norm": 1031.0673828125,
|
|
"regularization/mmd": 0.14609156548976898,
|
|
"regularization/rkhs_norm": 173.91854858398438,
|
|
"regularization/w1": 0.8660966753959656,
|
|
"rewards/chosen": 0.20830596282909683,
|
|
"rewards/margins": 0.1782946487024776,
|
|
"rewards/rejected": 0.030011314126619233,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.098224639892578,
|
|
"kl": 7.649110317230225,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -34977713.48242812,
|
|
"logits/rejected": -36509104.14678899,
|
|
"logps/chosen": -470.78973642172525,
|
|
"logps/rejected": -364.20943233944956,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.7403557300567627,
|
|
"metrics/advantage_var": 246.9385223388672,
|
|
"regularization/lipschitz_norm": 1009.1304931640625,
|
|
"regularization/mmd": 0.1441594511270523,
|
|
"regularization/rkhs_norm": 171.61839294433594,
|
|
"regularization/w1": 0.8476696014404297,
|
|
"rewards/chosen": 0.07622077746894032,
|
|
"rewards/margins": 0.2398331781888192,
|
|
"rewards/rejected": -0.16361240071987887,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 22.88916778564453,
|
|
"kl": 8.33399486541748,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34745693.053311795,
|
|
"logits/rejected": -36200144.36308623,
|
|
"logps/chosen": -495.8392568659128,
|
|
"logps/rejected": -364.34351361573374,
|
|
"loss": 0.5983,
|
|
"loss/base_kto": 3.7419095039367676,
|
|
"metrics/advantage_var": 240.70654296875,
|
|
"regularization/lipschitz_norm": 1065.1556396484375,
|
|
"regularization/mmd": 0.15013444423675537,
|
|
"regularization/rkhs_norm": 178.7314910888672,
|
|
"regularization/w1": 0.8947307467460632,
|
|
"rewards/chosen": 0.08383417206550069,
|
|
"rewards/margins": 0.238408914000365,
|
|
"rewards/rejected": -0.1545747419348643,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 19.463855743408203,
|
|
"kl": 7.266137599945068,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35963957.04866562,
|
|
"logits/rejected": -36229820.715396576,
|
|
"logps/chosen": -481.81171507064363,
|
|
"logps/rejected": -354.65685750388803,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.806239366531372,
|
|
"metrics/advantage_var": 234.2755126953125,
|
|
"regularization/lipschitz_norm": 1030.6988525390625,
|
|
"regularization/mmd": 0.1471906453371048,
|
|
"regularization/rkhs_norm": 175.22695922851562,
|
|
"regularization/w1": 0.8657870292663574,
|
|
"rewards/chosen": 0.04450525855718639,
|
|
"rewards/margins": 0.17270566497728748,
|
|
"rewards/rejected": -0.12820040642010108,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 16.1120548248291,
|
|
"kl": 5.923459529876709,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36382160.863492064,
|
|
"logits/rejected": -36331059.98769231,
|
|
"logps/chosen": -482.41001984126984,
|
|
"logps/rejected": -389.11834134615384,
|
|
"loss": 0.5892,
|
|
"loss/base_kto": 3.751215696334839,
|
|
"metrics/advantage_var": 213.98587036132812,
|
|
"regularization/lipschitz_norm": 978.8876953125,
|
|
"regularization/mmd": 0.14040164649486542,
|
|
"regularization/rkhs_norm": 167.1448211669922,
|
|
"regularization/w1": 0.822265625,
|
|
"rewards/chosen": -0.003133468022422185,
|
|
"rewards/margins": 0.23518008366027013,
|
|
"rewards/rejected": -0.2383135516826923,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 20.187463760375977,
|
|
"kl": 3.1745717525482178,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37297491.179810725,
|
|
"logits/rejected": -38113194.40247678,
|
|
"logps/chosen": -480.70174487381706,
|
|
"logps/rejected": -360.6059162151703,
|
|
"loss": 0.6016,
|
|
"loss/base_kto": 3.8223042488098145,
|
|
"metrics/advantage_var": 226.72300720214844,
|
|
"regularization/lipschitz_norm": 1004.8407592773438,
|
|
"regularization/mmd": 0.14660242199897766,
|
|
"regularization/rkhs_norm": 174.52670288085938,
|
|
"regularization/w1": 0.8440662622451782,
|
|
"rewards/chosen": -0.150642840268108,
|
|
"rewards/margins": 0.16019357379741542,
|
|
"rewards/rejected": -0.3108364140655234,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 20.050668716430664,
|
|
"kl": 9.79570484161377,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -34902970.469135806,
|
|
"logits/rejected": -35876883.44303797,
|
|
"logps/chosen": -455.43648726851853,
|
|
"logps/rejected": -377.20446004746833,
|
|
"loss": 0.6015,
|
|
"loss/base_kto": 3.7855632305145264,
|
|
"metrics/advantage_var": 250.9638214111328,
|
|
"regularization/lipschitz_norm": 1041.472412109375,
|
|
"regularization/mmd": 0.15120147168636322,
|
|
"regularization/rkhs_norm": 180.00173950195312,
|
|
"regularization/w1": 0.8748367428779602,
|
|
"rewards/chosen": 0.09103574870545188,
|
|
"rewards/margins": 0.19406917665615403,
|
|
"rewards/rejected": -0.10303342795070214,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 21.8386173248291,
|
|
"kl": 16.655717849731445,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -34515184.75725191,
|
|
"logits/rejected": -35051955.8144,
|
|
"logps/chosen": -464.76564885496185,
|
|
"logps/rejected": -355.4643,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.6980209350585938,
|
|
"metrics/advantage_var": 242.9734344482422,
|
|
"regularization/lipschitz_norm": 1039.7606201171875,
|
|
"regularization/mmd": 0.15043532848358154,
|
|
"regularization/rkhs_norm": 179.08969116210938,
|
|
"regularization/w1": 0.8733989000320435,
|
|
"rewards/chosen": 0.32243722231333494,
|
|
"rewards/margins": 0.27308180239145996,
|
|
"rewards/rejected": 0.049355419921875,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 21.20807647705078,
|
|
"kl": 15.139017105102539,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36116527.18894009,
|
|
"logits/rejected": -36718432.457869634,
|
|
"logps/chosen": -494.6800115207373,
|
|
"logps/rejected": -387.47717110492846,
|
|
"loss": 0.603,
|
|
"loss/base_kto": 3.7427093982696533,
|
|
"metrics/advantage_var": 287.2233581542969,
|
|
"regularization/lipschitz_norm": 1094.8924560546875,
|
|
"regularization/mmd": 0.16165395081043243,
|
|
"regularization/rkhs_norm": 192.44517517089844,
|
|
"regularization/w1": 0.9197096228599548,
|
|
"rewards/chosen": 0.3005254836309524,
|
|
"rewards/margins": 0.23888388148624104,
|
|
"rewards/rejected": 0.06164160214471135,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 19.53720474243164,
|
|
"kl": 22.42978858947754,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35044256.049155146,
|
|
"logits/rejected": -36973999.41494436,
|
|
"logps/chosen": -464.2317588325653,
|
|
"logps/rejected": -383.77931240063594,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.733435869216919,
|
|
"metrics/advantage_var": 259.2154235839844,
|
|
"regularization/lipschitz_norm": 1009.2023315429688,
|
|
"regularization/mmd": 0.1510559767484665,
|
|
"regularization/rkhs_norm": 179.82855224609375,
|
|
"regularization/w1": 0.8477298617362976,
|
|
"rewards/chosen": 0.30200598463301653,
|
|
"rewards/margins": 0.2217405937790134,
|
|
"rewards/rejected": 0.08026539085400312,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 17.993221282958984,
|
|
"kl": 8.867077827453613,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35755495.77287066,
|
|
"logits/rejected": -36845174.88544892,
|
|
"logps/chosen": -465.98945189274446,
|
|
"logps/rejected": -379.63191756965944,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.7875168323516846,
|
|
"metrics/advantage_var": 214.8356170654297,
|
|
"regularization/lipschitz_norm": 972.3076171875,
|
|
"regularization/mmd": 0.14126591384410858,
|
|
"regularization/rkhs_norm": 168.17369079589844,
|
|
"regularization/w1": 0.8167383074760437,
|
|
"rewards/chosen": 0.06241883238783394,
|
|
"rewards/margins": 0.18564694684368,
|
|
"rewards/rejected": -0.12322811445584607,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 24.116201400756836,
|
|
"kl": 17.905044555664062,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36314500.0913242,
|
|
"logits/rejected": -36291855.203852326,
|
|
"logps/chosen": -499.87661719939115,
|
|
"logps/rejected": -359.70856741573033,
|
|
"loss": 0.5796,
|
|
"loss/base_kto": 3.6291000843048096,
|
|
"metrics/advantage_var": 258.013671875,
|
|
"regularization/lipschitz_norm": 1020.1943359375,
|
|
"regularization/mmd": 0.15099453926086426,
|
|
"regularization/rkhs_norm": 179.75540161132812,
|
|
"regularization/w1": 0.8569631576538086,
|
|
"rewards/chosen": 0.3302828528928249,
|
|
"rewards/margins": 0.34321509911785436,
|
|
"rewards/rejected": -0.012932246225029469,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 33.473541259765625,
|
|
"kl": 21.3460693359375,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35796976.50832073,
|
|
"logits/rejected": -36226168.762520194,
|
|
"logps/chosen": -495.66442889561273,
|
|
"logps/rejected": -384.9128634894992,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.696981906890869,
|
|
"metrics/advantage_var": 294.5236511230469,
|
|
"regularization/lipschitz_norm": 1138.978271484375,
|
|
"regularization/mmd": 0.16508857905864716,
|
|
"regularization/rkhs_norm": 196.53404235839844,
|
|
"regularization/w1": 0.9567417502403259,
|
|
"rewards/chosen": 0.3779589816048719,
|
|
"rewards/margins": 0.29664941193329036,
|
|
"rewards/rejected": 0.08130956967158155,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.26300811767578,
|
|
"kl": 21.616954803466797,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36925939.358024694,
|
|
"logits/rejected": -37991884.151898734,
|
|
"logps/chosen": -503.7965374228395,
|
|
"logps/rejected": -367.091426028481,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.695453643798828,
|
|
"metrics/advantage_var": 244.77578735351562,
|
|
"regularization/lipschitz_norm": 1066.0438232421875,
|
|
"regularization/mmd": 0.15256932377815247,
|
|
"regularization/rkhs_norm": 181.630126953125,
|
|
"regularization/w1": 0.8954768180847168,
|
|
"rewards/chosen": 0.4246550195011092,
|
|
"rewards/margins": 0.2642496552834717,
|
|
"rewards/rejected": 0.16040536421763746,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 20.769075393676758,
|
|
"kl": 11.691008567810059,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36831582.81481481,
|
|
"logits/rejected": -37781082.73417722,
|
|
"logps/chosen": -524.2831790123457,
|
|
"logps/rejected": -366.26864121835445,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.6856117248535156,
|
|
"metrics/advantage_var": 238.05186462402344,
|
|
"regularization/lipschitz_norm": 1057.029541015625,
|
|
"regularization/mmd": 0.15006379783153534,
|
|
"regularization/rkhs_norm": 178.64736938476562,
|
|
"regularization/w1": 0.8879047632217407,
|
|
"rewards/chosen": 0.1865536254129292,
|
|
"rewards/margins": 0.2912244276621133,
|
|
"rewards/rejected": -0.10467080224918414,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 19.232282638549805,
|
|
"kl": 13.548723220825195,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -34448511.39527559,
|
|
"logits/rejected": -36557920.84341085,
|
|
"logps/chosen": -474.90871062992125,
|
|
"logps/rejected": -382.872238372093,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.714409589767456,
|
|
"metrics/advantage_var": 242.79356384277344,
|
|
"regularization/lipschitz_norm": 1022.59619140625,
|
|
"regularization/mmd": 0.15015900135040283,
|
|
"regularization/rkhs_norm": 178.76072692871094,
|
|
"regularization/w1": 0.8589809536933899,
|
|
"rewards/chosen": 0.27847155473363683,
|
|
"rewards/margins": 0.25291948228254063,
|
|
"rewards/rejected": 0.025552072451096173,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 24.14682960510254,
|
|
"kl": 9.039995193481445,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35461607.61904762,
|
|
"logits/rejected": -37804727.89269747,
|
|
"logps/chosen": -481.61473727422003,
|
|
"logps/rejected": -369.2634593889717,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.751467227935791,
|
|
"metrics/advantage_var": 252.6370849609375,
|
|
"regularization/lipschitz_norm": 1049.42431640625,
|
|
"regularization/mmd": 0.15219371020793915,
|
|
"regularization/rkhs_norm": 181.18301391601562,
|
|
"regularization/w1": 0.8815164566040039,
|
|
"rewards/chosen": 0.010386488120544133,
|
|
"rewards/margins": 0.2020670779584726,
|
|
"rewards/rejected": -0.19168058983792846,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 16.865453720092773,
|
|
"kl": 7.734155178070068,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -34780896.89719626,
|
|
"logits/rejected": -35494452.96551724,
|
|
"logps/chosen": -455.0786117601246,
|
|
"logps/rejected": -385.95273315047024,
|
|
"loss": 0.6081,
|
|
"loss/base_kto": 3.7895705699920654,
|
|
"metrics/advantage_var": 264.3589172363281,
|
|
"regularization/lipschitz_norm": 1095.6732177734375,
|
|
"regularization/mmd": 0.15475480258464813,
|
|
"regularization/rkhs_norm": 184.23191833496094,
|
|
"regularization/w1": 0.9203655123710632,
|
|
"rewards/chosen": 0.07494597865785023,
|
|
"rewards/margins": 0.2042531186614014,
|
|
"rewards/rejected": -0.12930714000355115,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 18.38845443725586,
|
|
"kl": 14.334578514099121,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35359368.42633229,
|
|
"logits/rejected": -37021437.60747664,
|
|
"logps/chosen": -494.8095121473354,
|
|
"logps/rejected": -377.56313278816197,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.746176242828369,
|
|
"metrics/advantage_var": 240.9512176513672,
|
|
"regularization/lipschitz_norm": 1032.4136962890625,
|
|
"regularization/mmd": 0.14993715286254883,
|
|
"regularization/rkhs_norm": 178.4966278076172,
|
|
"regularization/w1": 0.8672273755073547,
|
|
"rewards/chosen": 0.22056825930795698,
|
|
"rewards/margins": 0.22055451570238785,
|
|
"rewards/rejected": 1.3743605569144275e-05,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 22.69530487060547,
|
|
"kl": 7.90990686416626,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35112337.537007876,
|
|
"logits/rejected": -36394392.0124031,
|
|
"logps/chosen": -488.8363681102362,
|
|
"logps/rejected": -346.1170542635659,
|
|
"loss": 0.5947,
|
|
"loss/base_kto": 3.8013062477111816,
|
|
"metrics/advantage_var": 205.14932250976562,
|
|
"regularization/lipschitz_norm": 974.9683837890625,
|
|
"regularization/mmd": 0.13693474233150482,
|
|
"regularization/rkhs_norm": 163.01756286621094,
|
|
"regularization/w1": 0.8189733624458313,
|
|
"rewards/chosen": 0.026237824207215796,
|
|
"rewards/margins": 0.16785384115700988,
|
|
"rewards/rejected": -0.1416160169497941,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 21.700618743896484,
|
|
"kl": 5.915856838226318,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36401620.47604328,
|
|
"logits/rejected": -37539542.344391786,
|
|
"logps/chosen": -501.3424942040185,
|
|
"logps/rejected": -380.0257207740916,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.782858371734619,
|
|
"metrics/advantage_var": 250.19741821289062,
|
|
"regularization/lipschitz_norm": 1036.060791015625,
|
|
"regularization/mmd": 0.14888663589954376,
|
|
"regularization/rkhs_norm": 177.24598693847656,
|
|
"regularization/w1": 0.8702909350395203,
|
|
"rewards/chosen": 0.07331632351764755,
|
|
"rewards/margins": 0.21358003010458476,
|
|
"rewards/rejected": -0.1402637065869372,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 19.367273330688477,
|
|
"kl": 14.19812297821045,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36714834.01941747,
|
|
"logits/rejected": -37470514.27190332,
|
|
"logps/chosen": -492.8945691747573,
|
|
"logps/rejected": -386.22467428247734,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7310502529144287,
|
|
"metrics/advantage_var": 269.1242980957031,
|
|
"regularization/lipschitz_norm": 1073.406494140625,
|
|
"regularization/mmd": 0.15581868588924408,
|
|
"regularization/rkhs_norm": 185.49844360351562,
|
|
"regularization/w1": 0.9016615152359009,
|
|
"rewards/chosen": 0.13576719907495197,
|
|
"rewards/margins": 0.21811820368412044,
|
|
"rewards/rejected": -0.08235100460916847,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 14.528448104858398,
|
|
"kl": 10.813498497009277,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35483170.13333333,
|
|
"logits/rejected": -37314793.15692308,
|
|
"logps/chosen": -477.52708333333334,
|
|
"logps/rejected": -360.9183173076923,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.7103450298309326,
|
|
"metrics/advantage_var": 266.1695251464844,
|
|
"regularization/lipschitz_norm": 1050.378662109375,
|
|
"regularization/mmd": 0.1554114669561386,
|
|
"regularization/rkhs_norm": 185.01365661621094,
|
|
"regularization/w1": 0.8823180198669434,
|
|
"rewards/chosen": 0.026440117851136223,
|
|
"rewards/margins": 0.2533165733048622,
|
|
"rewards/rejected": -0.22687645545372595,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 18.759897232055664,
|
|
"kl": 4.533371925354004,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -34546959.741100326,
|
|
"logits/rejected": -35124968.72727273,
|
|
"logps/chosen": -468.8022855987055,
|
|
"logps/rejected": -388.53063446969696,
|
|
"loss": 0.5938,
|
|
"loss/base_kto": 3.6106491088867188,
|
|
"metrics/advantage_var": 319.78607177734375,
|
|
"regularization/lipschitz_norm": 1158.6326904296875,
|
|
"regularization/mmd": 0.16680000722408295,
|
|
"regularization/rkhs_norm": 198.5714569091797,
|
|
"regularization/w1": 0.9732513427734375,
|
|
"rewards/chosen": -0.010402836845916452,
|
|
"rewards/margins": 0.40733120523504945,
|
|
"rewards/rejected": -0.41773404208096593,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 17.778566360473633,
|
|
"kl": 8.78848934173584,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36056717.91730475,
|
|
"logits/rejected": -36985096.57416268,
|
|
"logps/chosen": -464.6512251148545,
|
|
"logps/rejected": -386.83268540669854,
|
|
"loss": 0.6112,
|
|
"loss/base_kto": 3.3962676525115967,
|
|
"metrics/advantage_var": 572.8226318359375,
|
|
"regularization/lipschitz_norm": 1521.9007568359375,
|
|
"regularization/mmd": 0.21491070091724396,
|
|
"regularization/rkhs_norm": 255.84609985351562,
|
|
"regularization/w1": 1.2783966064453125,
|
|
"rewards/chosen": 0.3514991281226072,
|
|
"rewards/margins": 0.6808625966639,
|
|
"rewards/rejected": -0.32936346854129284,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 19.549013137817383,
|
|
"kl": 13.9061918258667,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34939704.35220126,
|
|
"logits/rejected": -36651221.06832298,
|
|
"logps/chosen": -442.9081662735849,
|
|
"logps/rejected": -391.614494371118,
|
|
"loss": 0.6016,
|
|
"loss/base_kto": 3.3266537189483643,
|
|
"metrics/advantage_var": 532.240234375,
|
|
"regularization/lipschitz_norm": 1514.8626708984375,
|
|
"regularization/mmd": 0.2138865739107132,
|
|
"regularization/rkhs_norm": 254.62686157226562,
|
|
"regularization/w1": 1.2724847793579102,
|
|
"rewards/chosen": 0.5109047559822131,
|
|
"rewards/margins": 0.748533574280932,
|
|
"rewards/rejected": -0.23762881829871896,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 17.792404174804688,
|
|
"kl": 3.4995524883270264,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35142991.76508972,
|
|
"logits/rejected": -35455104.95952024,
|
|
"logps/chosen": -514.744545269168,
|
|
"logps/rejected": -366.4920352323838,
|
|
"loss": 0.6144,
|
|
"loss/base_kto": 3.391146183013916,
|
|
"metrics/advantage_var": 613.748046875,
|
|
"regularization/lipschitz_norm": 1552.6650390625,
|
|
"regularization/mmd": 0.21969342231750488,
|
|
"regularization/rkhs_norm": 261.539794921875,
|
|
"regularization/w1": 1.3042386770248413,
|
|
"rewards/chosen": 0.03567127769183957,
|
|
"rewards/margins": 0.6398779634114797,
|
|
"rewards/rejected": -0.6042066857196402,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 18.40005874633789,
|
|
"kl": 8.422476768493652,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36076973.728549145,
|
|
"logits/rejected": -36445333.03286385,
|
|
"logps/chosen": -461.4642648205928,
|
|
"logps/rejected": -374.2261345852895,
|
|
"loss": 0.6134,
|
|
"loss/base_kto": 3.3130805492401123,
|
|
"metrics/advantage_var": 659.9691772460938,
|
|
"regularization/lipschitz_norm": 1623.8614501953125,
|
|
"regularization/mmd": 0.22984924912452698,
|
|
"regularization/rkhs_norm": 273.6300048828125,
|
|
"regularization/w1": 1.3640435934066772,
|
|
"rewards/chosen": 0.31854762227598477,
|
|
"rewards/margins": 0.7969921351103695,
|
|
"rewards/rejected": -0.47844451283438477,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 18.619529724121094,
|
|
"kl": 11.051854133605957,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35030811.74961832,
|
|
"logits/rejected": -36247548.7232,
|
|
"logps/chosen": -462.54064885496183,
|
|
"logps/rejected": -370.33355,
|
|
"loss": 0.6078,
|
|
"loss/base_kto": 3.329913377761841,
|
|
"metrics/advantage_var": 571.1303100585938,
|
|
"regularization/lipschitz_norm": 1562.1920166015625,
|
|
"regularization/mmd": 0.22055485844612122,
|
|
"regularization/rkhs_norm": 262.5653076171875,
|
|
"regularization/w1": 1.3122414350509644,
|
|
"rewards/chosen": 0.34584243424976147,
|
|
"rewards/margins": 0.7301825709685115,
|
|
"rewards/rejected": -0.38434013671875,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 20.384653091430664,
|
|
"kl": 18.162588119506836,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36808110.01812689,
|
|
"logits/rejected": -36869875.57281554,
|
|
"logps/chosen": -507.0894070996979,
|
|
"logps/rejected": -381.21050768608416,
|
|
"loss": 0.6195,
|
|
"loss/base_kto": 3.315048933029175,
|
|
"metrics/advantage_var": 618.730712890625,
|
|
"regularization/lipschitz_norm": 1677.330078125,
|
|
"regularization/mmd": 0.23201961815357208,
|
|
"regularization/rkhs_norm": 276.2138366699219,
|
|
"regularization/w1": 1.4089572429656982,
|
|
"rewards/chosen": 0.5819440305773225,
|
|
"rewards/margins": 0.7377112429851969,
|
|
"rewards/rejected": -0.15576721240787444,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 15.094950675964355,
|
|
"kl": 5.735419750213623,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34146068.21052632,
|
|
"logits/rejected": -34838994.28571428,
|
|
"logps/chosen": -490.48118832236844,
|
|
"logps/rejected": -377.0299479166667,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.279555082321167,
|
|
"metrics/advantage_var": 617.1185302734375,
|
|
"regularization/lipschitz_norm": 1553.6285400390625,
|
|
"regularization/mmd": 0.2261827439069748,
|
|
"regularization/rkhs_norm": 269.26519775390625,
|
|
"regularization/w1": 1.3050479888916016,
|
|
"rewards/chosen": 0.28274867409153986,
|
|
"rewards/margins": 0.8007513204015286,
|
|
"rewards/rejected": -0.5180026463099888,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 18.639392852783203,
|
|
"kl": 7.991387844085693,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35504902.56410257,
|
|
"logits/rejected": -36857094.24390244,
|
|
"logps/chosen": -490.0498297275641,
|
|
"logps/rejected": -373.9388814786585,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.2353203296661377,
|
|
"metrics/advantage_var": 627.2642211914062,
|
|
"regularization/lipschitz_norm": 1591.6224365234375,
|
|
"regularization/mmd": 0.22979405522346497,
|
|
"regularization/rkhs_norm": 273.5643615722656,
|
|
"regularization/w1": 1.3369629383087158,
|
|
"rewards/chosen": 0.3509660378480569,
|
|
"rewards/margins": 0.8579493529204655,
|
|
"rewards/rejected": -0.5069833150724086,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 20.658008575439453,
|
|
"kl": 6.227613925933838,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -36547915.80775194,
|
|
"logits/rejected": -38035633.385826774,
|
|
"logps/chosen": -494.5046511627907,
|
|
"logps/rejected": -381.33375984251967,
|
|
"loss": 0.6224,
|
|
"loss/base_kto": 3.3543832302093506,
|
|
"metrics/advantage_var": 606.0228881835938,
|
|
"regularization/lipschitz_norm": 1661.4329833984375,
|
|
"regularization/mmd": 0.22887349128723145,
|
|
"regularization/rkhs_norm": 272.4684753417969,
|
|
"regularization/w1": 1.3956036567687988,
|
|
"rewards/chosen": 0.2094398439392563,
|
|
"rewards/margins": 0.7421111460738528,
|
|
"rewards/rejected": -0.5326713021345965,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 21.802608489990234,
|
|
"kl": 10.756336212158203,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35302135.22741433,
|
|
"logits/rejected": -36563354.88401254,
|
|
"logps/chosen": -476.05592873831773,
|
|
"logps/rejected": -394.12382445141066,
|
|
"loss": 0.6147,
|
|
"loss/base_kto": 3.2948577404022217,
|
|
"metrics/advantage_var": 664.4406127929688,
|
|
"regularization/lipschitz_norm": 1653.151611328125,
|
|
"regularization/mmd": 0.23432253301143646,
|
|
"regularization/rkhs_norm": 278.9554138183594,
|
|
"regularization/w1": 1.388647437095642,
|
|
"rewards/chosen": 0.39019937010197625,
|
|
"rewards/margins": 0.7748431639554489,
|
|
"rewards/rejected": -0.38464379385347275,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 16.32952117919922,
|
|
"kl": 14.149995803833008,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34725425.33965245,
|
|
"logits/rejected": -34796032.791344665,
|
|
"logps/chosen": -470.53297788309635,
|
|
"logps/rejected": -404.87053226429674,
|
|
"loss": 0.624,
|
|
"loss/base_kto": 3.3575081825256348,
|
|
"metrics/advantage_var": 635.5565795898438,
|
|
"regularization/lipschitz_norm": 1671.5345458984375,
|
|
"regularization/mmd": 0.2306479513645172,
|
|
"regularization/rkhs_norm": 274.5809020996094,
|
|
"regularization/w1": 1.404089093208313,
|
|
"rewards/chosen": 0.4045377372753752,
|
|
"rewards/margins": 0.716084664348646,
|
|
"rewards/rejected": -0.31154692707327086,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 14.87328815460205,
|
|
"kl": 12.064723014831543,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35413856.5748503,
|
|
"logits/rejected": -35093209.51633987,
|
|
"logps/chosen": -477.73409431137725,
|
|
"logps/rejected": -385.7476256127451,
|
|
"loss": 0.6032,
|
|
"loss/base_kto": 3.2641656398773193,
|
|
"metrics/advantage_var": 568.94287109375,
|
|
"regularization/lipschitz_norm": 1593.8173828125,
|
|
"regularization/mmd": 0.22247324883937836,
|
|
"regularization/rkhs_norm": 264.8490905761719,
|
|
"regularization/w1": 1.3388065099716187,
|
|
"rewards/chosen": 0.477585432772151,
|
|
"rewards/margins": 0.8137759992900993,
|
|
"rewards/rejected": -0.3361905665179483,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 26.503816604614258,
|
|
"kl": 20.650165557861328,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34678882.55037594,
|
|
"logits/rejected": -36262544.02601626,
|
|
"logps/chosen": -509.61287593984963,
|
|
"logps/rejected": -363.3782774390244,
|
|
"loss": 0.6127,
|
|
"loss/base_kto": 3.330998182296753,
|
|
"metrics/advantage_var": 603.1930541992188,
|
|
"regularization/lipschitz_norm": 1596.963134765625,
|
|
"regularization/mmd": 0.22921793162822723,
|
|
"regularization/rkhs_norm": 272.8785095214844,
|
|
"regularization/w1": 1.3414490222930908,
|
|
"rewards/chosen": 0.6033118685385338,
|
|
"rewards/margins": 0.714094249921918,
|
|
"rewards/rejected": -0.11078238138338414,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 14.60009479522705,
|
|
"kl": 15.492996215820312,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35416559.69426752,
|
|
"logits/rejected": -35820883.2392638,
|
|
"logps/chosen": -491.2158140923567,
|
|
"logps/rejected": -376.93153278374234,
|
|
"loss": 0.6125,
|
|
"loss/base_kto": 3.222810745239258,
|
|
"metrics/advantage_var": 688.3303833007812,
|
|
"regularization/lipschitz_norm": 1711.7630615234375,
|
|
"regularization/mmd": 0.23897671699523926,
|
|
"regularization/rkhs_norm": 284.49609375,
|
|
"regularization/w1": 1.4378809928894043,
|
|
"rewards/chosen": 0.6643336107776423,
|
|
"rewards/margins": 0.8913463854732904,
|
|
"rewards/rejected": -0.227012774695648,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 19.115020751953125,
|
|
"kl": 14.2912015914917,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -36602624.40699523,
|
|
"logits/rejected": -38045623.64362519,
|
|
"logps/chosen": -504.0611089030207,
|
|
"logps/rejected": -366.39544930875576,
|
|
"loss": 0.5952,
|
|
"loss/base_kto": 3.26883864402771,
|
|
"metrics/advantage_var": 534.1740112304688,
|
|
"regularization/lipschitz_norm": 1521.0423583984375,
|
|
"regularization/mmd": 0.2153901606798172,
|
|
"regularization/rkhs_norm": 256.4168395996094,
|
|
"regularization/w1": 1.2776756286621094,
|
|
"rewards/chosen": 0.4696582497019078,
|
|
"rewards/margins": 0.759111430955508,
|
|
"rewards/rejected": -0.28945318125360026,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 13.423221588134766,
|
|
"kl": 11.066169738769531,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34700488.971962616,
|
|
"logits/rejected": -36084604.38871473,
|
|
"logps/chosen": -470.60153816199374,
|
|
"logps/rejected": -377.42750783699057,
|
|
"loss": 0.6023,
|
|
"loss/base_kto": 3.35111403465271,
|
|
"metrics/advantage_var": 568.9474487304688,
|
|
"regularization/lipschitz_norm": 1493.8076171875,
|
|
"regularization/mmd": 0.21226052939891815,
|
|
"regularization/rkhs_norm": 252.6911163330078,
|
|
"regularization/w1": 1.2547982931137085,
|
|
"rewards/chosen": 0.3544506417628018,
|
|
"rewards/margins": 0.6904167460131446,
|
|
"rewards/rejected": -0.33596610425034285,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 22.768653869628906,
|
|
"kl": 8.966629981994629,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35089128.426494345,
|
|
"logits/rejected": -36029642.94099849,
|
|
"logps/chosen": -471.4241720516963,
|
|
"logps/rejected": -378.2768532526475,
|
|
"loss": 0.5958,
|
|
"loss/base_kto": 3.3208909034729004,
|
|
"metrics/advantage_var": 496.6685485839844,
|
|
"regularization/lipschitz_norm": 1471.4638671875,
|
|
"regularization/mmd": 0.20927010476589203,
|
|
"regularization/rkhs_norm": 249.13108825683594,
|
|
"regularization/w1": 1.2360297441482544,
|
|
"rewards/chosen": 0.318765027072395,
|
|
"rewards/margins": 0.7169578324217709,
|
|
"rewards/rejected": -0.39819280534937596,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.214309692382812,
|
|
"kl": 8.788565635681152,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34333707.39586645,
|
|
"logits/rejected": -35671485.14900154,
|
|
"logps/chosen": -486.2912857710652,
|
|
"logps/rejected": -376.6211357526882,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.2602412700653076,
|
|
"metrics/advantage_var": 572.3546752929688,
|
|
"regularization/lipschitz_norm": 1548.5469970703125,
|
|
"regularization/mmd": 0.21799610555171967,
|
|
"regularization/rkhs_norm": 259.5191955566406,
|
|
"regularization/w1": 1.3007795810699463,
|
|
"rewards/chosen": 0.3632470936009539,
|
|
"rewards/margins": 0.8114203704406717,
|
|
"rewards/rejected": -0.44817327683971775,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.935007095336914,
|
|
"kl": 10.534090042114258,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34666082.429725364,
|
|
"logits/rejected": -36020598.89863843,
|
|
"logps/chosen": -483.5410945072698,
|
|
"logps/rejected": -373.60795669440245,
|
|
"loss": 0.6119,
|
|
"loss/base_kto": 3.2726731300354004,
|
|
"metrics/advantage_var": 650.3192749023438,
|
|
"regularization/lipschitz_norm": 1655.721923828125,
|
|
"regularization/mmd": 0.23140673339366913,
|
|
"regularization/rkhs_norm": 275.4842224121094,
|
|
"regularization/w1": 1.3908065557479858,
|
|
"rewards/chosen": 0.39531334305425836,
|
|
"rewards/margins": 0.7906553633862601,
|
|
"rewards/rejected": -0.3953420203320017,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 18.949359893798828,
|
|
"kl": 6.50945520401001,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -33921636.4244373,
|
|
"logits/rejected": -35300809.5319149,
|
|
"logps/chosen": -500.0674236334405,
|
|
"logps/rejected": -346.7652688069909,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.265681505203247,
|
|
"metrics/advantage_var": 583.9192504882812,
|
|
"regularization/lipschitz_norm": 1563.104248046875,
|
|
"regularization/mmd": 0.22389140725135803,
|
|
"regularization/rkhs_norm": 266.53741455078125,
|
|
"regularization/w1": 1.3130074739456177,
|
|
"rewards/chosen": 0.3065753703715334,
|
|
"rewards/margins": 0.8083490538673302,
|
|
"rewards/rejected": -0.5017736834957969,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 17.29044532775879,
|
|
"kl": 11.243669509887695,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33964555.4115899,
|
|
"logits/rejected": -36896923.202635914,
|
|
"logps/chosen": -503.8058135215453,
|
|
"logps/rejected": -364.0481363261944,
|
|
"loss": 0.6147,
|
|
"loss/base_kto": 3.266183614730835,
|
|
"metrics/advantage_var": 679.18994140625,
|
|
"regularization/lipschitz_norm": 1679.4996337890625,
|
|
"regularization/mmd": 0.24045300483703613,
|
|
"regularization/rkhs_norm": 286.25360107421875,
|
|
"regularization/w1": 1.4107798337936401,
|
|
"rewards/chosen": 0.5210249335415119,
|
|
"rewards/margins": 0.8321277613897924,
|
|
"rewards/rejected": -0.31110282784828047,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 18.00981330871582,
|
|
"kl": 8.955912590026855,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34107723.763239875,
|
|
"logits/rejected": -35853202.85893417,
|
|
"logps/chosen": -464.2046339563863,
|
|
"logps/rejected": -376.59502351097177,
|
|
"loss": 0.6164,
|
|
"loss/base_kto": 3.2926902770996094,
|
|
"metrics/advantage_var": 973.8168334960938,
|
|
"regularization/lipschitz_norm": 1667.1356201171875,
|
|
"regularization/mmd": 0.23807401955127716,
|
|
"regularization/rkhs_norm": 283.42144775390625,
|
|
"regularization/w1": 1.4003939628601074,
|
|
"rewards/chosen": 0.4130081699644665,
|
|
"rewards/margins": 0.761906265361459,
|
|
"rewards/rejected": -0.34889809539699257,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 20.81473731994629,
|
|
"kl": 13.247878074645996,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34005499.123809524,
|
|
"logits/rejected": -36260350.42461538,
|
|
"logps/chosen": -508.7732638888889,
|
|
"logps/rejected": -376.1651923076923,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.281367063522339,
|
|
"metrics/advantage_var": 645.3544921875,
|
|
"regularization/lipschitz_norm": 1575.2410888671875,
|
|
"regularization/mmd": 0.232137069106102,
|
|
"regularization/rkhs_norm": 276.3536071777344,
|
|
"regularization/w1": 1.3232024908065796,
|
|
"rewards/chosen": 0.45817687019469244,
|
|
"rewards/margins": 0.7560264138395242,
|
|
"rewards/rejected": -0.29784954364483174,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.589284896850586,
|
|
"kl": 9.518244743347168,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -33770334.82988871,
|
|
"logits/rejected": -35405043.809523806,
|
|
"logps/chosen": -484.3436009538951,
|
|
"logps/rejected": -356.1745871735791,
|
|
"loss": 0.6135,
|
|
"loss/base_kto": 3.3802649974823,
|
|
"metrics/advantage_var": 610.53466796875,
|
|
"regularization/lipschitz_norm": 1561.3052978515625,
|
|
"regularization/mmd": 0.2164938896894455,
|
|
"regularization/rkhs_norm": 257.7308044433594,
|
|
"regularization/w1": 1.311496615409851,
|
|
"rewards/chosen": 0.30456038385582274,
|
|
"rewards/margins": 0.6745935894184787,
|
|
"rewards/rejected": -0.370033205562656,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 19.481111526489258,
|
|
"kl": 8.323575019836426,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34992025.1177394,
|
|
"logits/rejected": -35880778.45101088,
|
|
"logps/chosen": -473.9120879120879,
|
|
"logps/rejected": -369.423551710731,
|
|
"loss": 0.6238,
|
|
"loss/base_kto": 3.34150767326355,
|
|
"metrics/advantage_var": 633.9444580078125,
|
|
"regularization/lipschitz_norm": 1686.3128662109375,
|
|
"regularization/mmd": 0.2320529669523239,
|
|
"regularization/rkhs_norm": 276.2535400390625,
|
|
"regularization/w1": 1.416502833366394,
|
|
"rewards/chosen": 0.33816430108326384,
|
|
"rewards/margins": 0.7282971280114812,
|
|
"rewards/rejected": -0.3901328269282173,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 15.917203903198242,
|
|
"kl": 14.068951606750488,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33293421.20556414,
|
|
"logits/rejected": -34839262.4328594,
|
|
"logps/chosen": -470.7300521638331,
|
|
"logps/rejected": -367.746198657188,
|
|
"loss": 0.609,
|
|
"loss/base_kto": 3.3531925678253174,
|
|
"metrics/advantage_var": 532.55517578125,
|
|
"regularization/lipschitz_norm": 1554.1697998046875,
|
|
"regularization/mmd": 0.21358957886695862,
|
|
"regularization/rkhs_norm": 254.27330017089844,
|
|
"regularization/w1": 1.3055025339126587,
|
|
"rewards/chosen": 0.414887983979545,
|
|
"rewards/margins": 0.6779400098453379,
|
|
"rewards/rejected": -0.26305202586579285,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 14.823602676391602,
|
|
"kl": 10.622819900512695,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34610425.35064935,
|
|
"logits/rejected": -35481131.18072289,
|
|
"logps/chosen": -484.32112418831167,
|
|
"logps/rejected": -387.072218561747,
|
|
"loss": 0.598,
|
|
"loss/base_kto": 3.270538330078125,
|
|
"metrics/advantage_var": 601.8978881835938,
|
|
"regularization/lipschitz_norm": 1537.4549560546875,
|
|
"regularization/mmd": 0.22194503247737885,
|
|
"regularization/rkhs_norm": 264.22027587890625,
|
|
"regularization/w1": 1.2914621829986572,
|
|
"rewards/chosen": 0.4453907756062297,
|
|
"rewards/margins": 0.8184043983957275,
|
|
"rewards/rejected": -0.37301362278949785,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 15.449972152709961,
|
|
"kl": 6.428173065185547,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34536957.64777948,
|
|
"logits/rejected": -35821342.62200957,
|
|
"logps/chosen": -484.12557427258804,
|
|
"logps/rejected": -375.1648225677831,
|
|
"loss": 0.6036,
|
|
"loss/base_kto": 3.31561541557312,
|
|
"metrics/advantage_var": 600.2677612304688,
|
|
"regularization/lipschitz_norm": 1538.493896484375,
|
|
"regularization/mmd": 0.22086206078529358,
|
|
"regularization/rkhs_norm": 262.9310302734375,
|
|
"regularization/w1": 1.2923349142074585,
|
|
"rewards/chosen": 0.31995327585871697,
|
|
"rewards/margins": 0.7706536129051683,
|
|
"rewards/rejected": -0.45070033704645135,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 20.079334259033203,
|
|
"kl": 13.06872272491455,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33904351.01840491,
|
|
"logits/rejected": -35549324.22929937,
|
|
"logps/chosen": -484.3055023006135,
|
|
"logps/rejected": -382.65057722929936,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.2971794605255127,
|
|
"metrics/advantage_var": 488.62335205078125,
|
|
"regularization/lipschitz_norm": 1478.4853515625,
|
|
"regularization/mmd": 0.21024754643440247,
|
|
"regularization/rkhs_norm": 250.29470825195312,
|
|
"regularization/w1": 1.2419277429580688,
|
|
"rewards/chosen": 0.44681623962027894,
|
|
"rewards/margins": 0.7049084759872799,
|
|
"rewards/rejected": -0.2580922363670009,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 22.235336303710938,
|
|
"kl": 14.48465633392334,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34517573.9277865,
|
|
"logits/rejected": -35482412.19284604,
|
|
"logps/chosen": -474.99617346938777,
|
|
"logps/rejected": -393.0787811041991,
|
|
"loss": 0.6048,
|
|
"loss/base_kto": 3.3372159004211426,
|
|
"metrics/advantage_var": 549.857421875,
|
|
"regularization/lipschitz_norm": 1529.2337646484375,
|
|
"regularization/mmd": 0.21660156548023224,
|
|
"regularization/rkhs_norm": 257.8589782714844,
|
|
"regularization/w1": 1.2845563888549805,
|
|
"rewards/chosen": 0.3548815381395948,
|
|
"rewards/margins": 0.709813264194829,
|
|
"rewards/rejected": -0.35493172605523426,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 18.25558090209961,
|
|
"kl": 10.816429138183594,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34766032.50678733,
|
|
"logits/rejected": -35377693.04376013,
|
|
"logps/chosen": -474.4428261689291,
|
|
"logps/rejected": -350.2752228525122,
|
|
"loss": 0.6127,
|
|
"loss/base_kto": 3.3443429470062256,
|
|
"metrics/advantage_var": 577.3859252929688,
|
|
"regularization/lipschitz_norm": 1594.7008056640625,
|
|
"regularization/mmd": 0.21744489669799805,
|
|
"regularization/rkhs_norm": 258.86297607421875,
|
|
"regularization/w1": 1.3395487070083618,
|
|
"rewards/chosen": 0.39128736493094834,
|
|
"rewards/margins": 0.7063987199628567,
|
|
"rewards/rejected": -0.3151113550319084,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 16.051433563232422,
|
|
"kl": 8.202681541442871,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34601436.88676236,
|
|
"logits/rejected": -35412253.402756505,
|
|
"logps/chosen": -485.18620414673046,
|
|
"logps/rejected": -359.6713246554364,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.3309967517852783,
|
|
"metrics/advantage_var": 613.6174926757812,
|
|
"regularization/lipschitz_norm": 1523.8358154296875,
|
|
"regularization/mmd": 0.21878688037395477,
|
|
"regularization/rkhs_norm": 260.4605712890625,
|
|
"regularization/w1": 1.2800220251083374,
|
|
"rewards/chosen": 0.375434692966881,
|
|
"rewards/margins": 0.754728768574952,
|
|
"rewards/rejected": -0.37929407560807094,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 20.815223693847656,
|
|
"kl": 11.590879440307617,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -32725838.25499232,
|
|
"logits/rejected": -33893693.45627981,
|
|
"logps/chosen": -476.3749039938556,
|
|
"logps/rejected": -352.7370578298887,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.261859178543091,
|
|
"metrics/advantage_var": 574.8982543945312,
|
|
"regularization/lipschitz_norm": 1554.3128662109375,
|
|
"regularization/mmd": 0.22218680381774902,
|
|
"regularization/rkhs_norm": 264.5081481933594,
|
|
"regularization/w1": 1.305622935295105,
|
|
"rewards/chosen": 0.47808050340221775,
|
|
"rewards/margins": 0.8157705080754342,
|
|
"rewards/rejected": -0.3376900046732164,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.088031768798828,
|
|
"kl": 11.878664016723633,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34259679.50464396,
|
|
"logits/rejected": -35375110.460567825,
|
|
"logps/chosen": -469.6266447368421,
|
|
"logps/rejected": -393.8571322949527,
|
|
"loss": 0.6088,
|
|
"loss/base_kto": 3.3371567726135254,
|
|
"metrics/advantage_var": 551.409423828125,
|
|
"regularization/lipschitz_norm": 1562.9835205078125,
|
|
"regularization/mmd": 0.2202134132385254,
|
|
"regularization/rkhs_norm": 262.1588439941406,
|
|
"regularization/w1": 1.3129061460494995,
|
|
"rewards/chosen": 0.4428910293815306,
|
|
"rewards/margins": 0.7090021445573734,
|
|
"rewards/rejected": -0.26611111517584285,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 21.351125717163086,
|
|
"kl": 11.486132621765137,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34189819.27384616,
|
|
"logits/rejected": -35112222.06984127,
|
|
"logps/chosen": -495.61538461538464,
|
|
"logps/rejected": -391.8570436507936,
|
|
"loss": 0.6038,
|
|
"loss/base_kto": 3.287785768508911,
|
|
"metrics/advantage_var": 554.07470703125,
|
|
"regularization/lipschitz_norm": 1572.2630615234375,
|
|
"regularization/mmd": 0.22212015092372894,
|
|
"regularization/rkhs_norm": 264.4287109375,
|
|
"regularization/w1": 1.320701003074646,
|
|
"rewards/chosen": 0.42416682316706733,
|
|
"rewards/margins": 0.7961476658785009,
|
|
"rewards/rejected": -0.37198084271143356,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 16.664878845214844,
|
|
"kl": 13.020438194274902,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34901265.93630573,
|
|
"logits/rejected": -37257592.93251534,
|
|
"logps/chosen": -474.5579219745223,
|
|
"logps/rejected": -370.8956096625767,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.231579303741455,
|
|
"metrics/advantage_var": 534.8076171875,
|
|
"regularization/lipschitz_norm": 1537.1300048828125,
|
|
"regularization/mmd": 0.21699519455432892,
|
|
"regularization/rkhs_norm": 258.3276062011719,
|
|
"regularization/w1": 1.291189193725586,
|
|
"rewards/chosen": 0.5680030774159036,
|
|
"rewards/margins": 0.8133468532528716,
|
|
"rewards/rejected": -0.24534377583696798,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 18.43654441833496,
|
|
"kl": 13.266098022460938,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33752704.0,
|
|
"logits/rejected": -34829000.64864865,
|
|
"logps/chosen": -477.6653797238372,
|
|
"logps/rejected": -375.7274070945946,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.269211530685425,
|
|
"metrics/advantage_var": 561.1244506835938,
|
|
"regularization/lipschitz_norm": 1529.3062744140625,
|
|
"regularization/mmd": 0.21642573177814484,
|
|
"regularization/rkhs_norm": 257.649658203125,
|
|
"regularization/w1": 1.284617304801941,
|
|
"rewards/chosen": 0.5461674180141715,
|
|
"rewards/margins": 0.8230417540506474,
|
|
"rewards/rejected": -0.27687433603647593,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 18.059795379638672,
|
|
"kl": 15.921954154968262,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35747996.28616852,
|
|
"logits/rejected": -36362991.877112135,
|
|
"logps/chosen": -485.92259538950714,
|
|
"logps/rejected": -389.21193356374806,
|
|
"loss": 0.5947,
|
|
"loss/base_kto": 3.2594032287597656,
|
|
"metrics/advantage_var": 546.0824584960938,
|
|
"regularization/lipschitz_norm": 1527.6932373046875,
|
|
"regularization/mmd": 0.21503780782222748,
|
|
"regularization/rkhs_norm": 255.9973602294922,
|
|
"regularization/w1": 1.2832622528076172,
|
|
"rewards/chosen": 0.5455893317921552,
|
|
"rewards/margins": 0.8161443035603484,
|
|
"rewards/rejected": -0.27055497176819315,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 18.50031089782715,
|
|
"kl": 11.904764175415039,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -33491071.163398694,
|
|
"logits/rejected": -34679386.71471471,
|
|
"logps/chosen": -488.5717422385621,
|
|
"logps/rejected": -376.6834177927928,
|
|
"loss": 0.5673,
|
|
"loss/base_kto": 3.265979051589966,
|
|
"metrics/advantage_var": 435.43310546875,
|
|
"regularization/lipschitz_norm": 1286.2176513671875,
|
|
"regularization/mmd": 0.191904217004776,
|
|
"regularization/rkhs_norm": 228.45742797851562,
|
|
"regularization/w1": 1.0804227590560913,
|
|
"rewards/chosen": 0.41821299035564746,
|
|
"rewards/margins": 0.7594718670301736,
|
|
"rewards/rejected": -0.3412588766745261,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 18.242860794067383,
|
|
"kl": 8.851688385009766,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -33713933.99113737,
|
|
"logits/rejected": -34132294.049751244,
|
|
"logps/chosen": -459.3748153618907,
|
|
"logps/rejected": -383.0402155887231,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.2155025005340576,
|
|
"metrics/advantage_var": 463.7978515625,
|
|
"regularization/lipschitz_norm": 1314.51904296875,
|
|
"regularization/mmd": 0.19804874062538147,
|
|
"regularization/rkhs_norm": 235.77232360839844,
|
|
"regularization/w1": 1.1041959524154663,
|
|
"rewards/chosen": 0.43894311631623895,
|
|
"rewards/margins": 0.8680322592247434,
|
|
"rewards/rejected": -0.42908914290850436,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 14.94932746887207,
|
|
"kl": 14.95853328704834,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -32988485.2327044,
|
|
"logits/rejected": -35565593.440993786,
|
|
"logps/chosen": -498.71555621069183,
|
|
"logps/rejected": -362.0689538043478,
|
|
"loss": 0.5556,
|
|
"loss/base_kto": 3.146251916885376,
|
|
"metrics/advantage_var": 433.46002197265625,
|
|
"regularization/lipschitz_norm": 1312.9205322265625,
|
|
"regularization/mmd": 0.19606690108776093,
|
|
"regularization/rkhs_norm": 233.4129638671875,
|
|
"regularization/w1": 1.1028531789779663,
|
|
"rewards/chosen": 0.6007800791998329,
|
|
"rewards/margins": 0.8850090971327957,
|
|
"rewards/rejected": -0.2842290179329629,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 16.971399307250977,
|
|
"kl": 17.06403350830078,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -33604675.43032159,
|
|
"logits/rejected": -35877353.95215311,
|
|
"logps/chosen": -482.5694391271057,
|
|
"logps/rejected": -385.4413875598086,
|
|
"loss": 0.556,
|
|
"loss/base_kto": 3.1591320037841797,
|
|
"metrics/advantage_var": 440.32861328125,
|
|
"regularization/lipschitz_norm": 1301.7017822265625,
|
|
"regularization/mmd": 0.19571247696876526,
|
|
"regularization/rkhs_norm": 232.9910430908203,
|
|
"regularization/w1": 1.093429446220398,
|
|
"rewards/chosen": 0.6331080952245645,
|
|
"rewards/margins": 0.8811377888767524,
|
|
"rewards/rejected": -0.248029693652188,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 15.447782516479492,
|
|
"kl": 14.525923728942871,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34164105.84615385,
|
|
"logits/rejected": -34123059.359253496,
|
|
"logps/chosen": -482.33330062794346,
|
|
"logps/rejected": -374.1341125583204,
|
|
"loss": 0.5665,
|
|
"loss/base_kto": 3.206890821456909,
|
|
"metrics/advantage_var": 435.4579162597656,
|
|
"regularization/lipschitz_norm": 1344.9830322265625,
|
|
"regularization/mmd": 0.1954275369644165,
|
|
"regularization/rkhs_norm": 232.6518096923828,
|
|
"regularization/w1": 1.1297857761383057,
|
|
"rewards/chosen": 0.5523423894046802,
|
|
"rewards/margins": 0.8329416245952541,
|
|
"rewards/rejected": -0.28059923519057395,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.84161949157715,
|
|
"kl": 14.022412300109863,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33639749.81818182,
|
|
"logits/rejected": -34408898.313253015,
|
|
"logps/chosen": -472.515726461039,
|
|
"logps/rejected": -370.6802757906627,
|
|
"loss": 0.5624,
|
|
"loss/base_kto": 3.1868457794189453,
|
|
"metrics/advantage_var": 410.8081970214844,
|
|
"regularization/lipschitz_norm": 1333.5665283203125,
|
|
"regularization/mmd": 0.1918194591999054,
|
|
"regularization/rkhs_norm": 228.3564910888672,
|
|
"regularization/w1": 1.1201958656311035,
|
|
"rewards/chosen": 0.5040625039633219,
|
|
"rewards/margins": 0.8412723893422398,
|
|
"rewards/rejected": -0.337209885378918,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.6634464263916,
|
|
"kl": 12.651629447937012,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33675174.95652174,
|
|
"logits/rejected": -36437268.93081761,
|
|
"logps/chosen": -489.09171195652175,
|
|
"logps/rejected": -361.3551985062893,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.1931161880493164,
|
|
"metrics/advantage_var": 440.7278747558594,
|
|
"regularization/lipschitz_norm": 1350.2635498046875,
|
|
"regularization/mmd": 0.19630078971385956,
|
|
"regularization/rkhs_norm": 233.69140625,
|
|
"regularization/w1": 1.1342214345932007,
|
|
"rewards/chosen": 0.5269959254294448,
|
|
"rewards/margins": 0.8607994501905248,
|
|
"rewards/rejected": -0.33380352476108,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.905786514282227,
|
|
"kl": 13.652798652648926,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -33972603.37833595,
|
|
"logits/rejected": -36312052.852255054,
|
|
"logps/chosen": -524.2321428571429,
|
|
"logps/rejected": -371.7416650466563,
|
|
"loss": 0.5547,
|
|
"loss/base_kto": 3.133366346359253,
|
|
"metrics/advantage_var": 459.79296875,
|
|
"regularization/lipschitz_norm": 1314.7943115234375,
|
|
"regularization/mmd": 0.19959811866283417,
|
|
"regularization/rkhs_norm": 237.6168212890625,
|
|
"regularization/w1": 1.1044272184371948,
|
|
"rewards/chosen": 0.5978972286781299,
|
|
"rewards/margins": 0.9215670656924305,
|
|
"rewards/rejected": -0.32366983701430063,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 13.987251281738281,
|
|
"kl": 13.4874906539917,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33498057.28244275,
|
|
"logits/rejected": -35428125.9008,
|
|
"logps/chosen": -473.863358778626,
|
|
"logps/rejected": -391.28785,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.1569135189056396,
|
|
"metrics/advantage_var": 447.26593017578125,
|
|
"regularization/lipschitz_norm": 1351.6658935546875,
|
|
"regularization/mmd": 0.20029079914093018,
|
|
"regularization/rkhs_norm": 238.44143676757812,
|
|
"regularization/w1": 1.135399341583252,
|
|
"rewards/chosen": 0.5698268278864503,
|
|
"rewards/margins": 0.8870999968317628,
|
|
"rewards/rejected": -0.3172731689453125,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 24.192644119262695,
|
|
"kl": 8.75890827178955,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33442620.58015267,
|
|
"logits/rejected": -35251050.9056,
|
|
"logps/chosen": -492.8916984732824,
|
|
"logps/rejected": -365.31045,
|
|
"loss": 0.5596,
|
|
"loss/base_kto": 3.1864590644836426,
|
|
"metrics/advantage_var": 441.95703125,
|
|
"regularization/lipschitz_norm": 1303.9627685546875,
|
|
"regularization/mmd": 0.19536092877388,
|
|
"regularization/rkhs_norm": 232.572509765625,
|
|
"regularization/w1": 1.095328688621521,
|
|
"rewards/chosen": 0.5029198100548664,
|
|
"rewards/margins": 0.8839807231408039,
|
|
"rewards/rejected": -0.3810609130859375,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 13.031659126281738,
|
|
"kl": 14.070281028747559,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -33599494.27258805,
|
|
"logits/rejected": -34143909.767145135,
|
|
"logps/chosen": -443.59355857580397,
|
|
"logps/rejected": -372.660735645933,
|
|
"loss": 0.551,
|
|
"loss/base_kto": 3.18375563621521,
|
|
"metrics/advantage_var": 358.3189392089844,
|
|
"regularization/lipschitz_norm": 1239.441650390625,
|
|
"regularization/mmd": 0.1828051209449768,
|
|
"regularization/rkhs_norm": 217.62515258789062,
|
|
"regularization/w1": 1.0411310195922852,
|
|
"rewards/chosen": 0.5493479987198507,
|
|
"rewards/margins": 0.8359266014878681,
|
|
"rewards/rejected": -0.28657860276801733,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 16.931217193603516,
|
|
"kl": 15.399487495422363,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34551310.35514019,
|
|
"logits/rejected": -35247601.55485894,
|
|
"logps/chosen": -487.81580996884736,
|
|
"logps/rejected": -392.4702929075235,
|
|
"loss": 0.559,
|
|
"loss/base_kto": 3.1606571674346924,
|
|
"metrics/advantage_var": 443.51495361328125,
|
|
"regularization/lipschitz_norm": 1328.6239013671875,
|
|
"regularization/mmd": 0.19551421701908112,
|
|
"regularization/rkhs_norm": 232.75503540039062,
|
|
"regularization/w1": 1.116044044494629,
|
|
"rewards/chosen": 0.559925970630111,
|
|
"rewards/margins": 0.8630176117602295,
|
|
"rewards/rejected": -0.30309164113011855,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 18.24913215637207,
|
|
"kl": 12.832463264465332,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35501554.46345256,
|
|
"logits/rejected": -34190289.381475665,
|
|
"logps/chosen": -489.16854587869364,
|
|
"logps/rejected": -367.9146144034537,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.201253652572632,
|
|
"metrics/advantage_var": 400.7730407714844,
|
|
"regularization/lipschitz_norm": 1306.8446044921875,
|
|
"regularization/mmd": 0.18958069384098053,
|
|
"regularization/rkhs_norm": 225.69131469726562,
|
|
"regularization/w1": 1.0977494716644287,
|
|
"rewards/chosen": 0.5137816528510157,
|
|
"rewards/margins": 0.8428591653914886,
|
|
"rewards/rejected": -0.3290775125404729,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 17.598134994506836,
|
|
"kl": 11.90916633605957,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33998500.716323294,
|
|
"logits/rejected": -35748152.40677966,
|
|
"logps/chosen": -464.4553783676704,
|
|
"logps/rejected": -355.8242969953775,
|
|
"loss": 0.5528,
|
|
"loss/base_kto": 3.1689717769622803,
|
|
"metrics/advantage_var": 385.138427734375,
|
|
"regularization/lipschitz_norm": 1268.771728515625,
|
|
"regularization/mmd": 0.18765221536159515,
|
|
"regularization/rkhs_norm": 223.39549255371094,
|
|
"regularization/w1": 1.0657681226730347,
|
|
"rewards/chosen": 0.5189987158435272,
|
|
"rewards/margins": 0.86975529785973,
|
|
"rewards/rejected": -0.3507565820162028,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 21.09761619567871,
|
|
"kl": 10.740870475769043,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34534776.92259084,
|
|
"logits/rejected": -35369858.96754251,
|
|
"logps/chosen": -455.9173578199052,
|
|
"logps/rejected": -396.7879153786708,
|
|
"loss": 0.5521,
|
|
"loss/base_kto": 3.195344924926758,
|
|
"metrics/advantage_var": 408.6285705566406,
|
|
"regularization/lipschitz_norm": 1231.0389404296875,
|
|
"regularization/mmd": 0.1872580349445343,
|
|
"regularization/rkhs_norm": 222.9262237548828,
|
|
"regularization/w1": 1.034072756767273,
|
|
"rewards/chosen": 0.4403811389989386,
|
|
"rewards/margins": 0.8548842807529475,
|
|
"rewards/rejected": -0.4145031417540089,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 17.74950408935547,
|
|
"kl": 10.207816123962402,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35081552.02488336,
|
|
"logits/rejected": -34916361.64521193,
|
|
"logps/chosen": -471.7910672628305,
|
|
"logps/rejected": -393.3380592621664,
|
|
"loss": 0.5677,
|
|
"loss/base_kto": 3.2030608654022217,
|
|
"metrics/advantage_var": 436.63531494140625,
|
|
"regularization/lipschitz_norm": 1359.7098388671875,
|
|
"regularization/mmd": 0.19673502445220947,
|
|
"regularization/rkhs_norm": 234.2083740234375,
|
|
"regularization/w1": 1.1421562433242798,
|
|
"rewards/chosen": 0.4566846405551371,
|
|
"rewards/margins": 0.8489532301198692,
|
|
"rewards/rejected": -0.39226858956473215,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 15.340740203857422,
|
|
"kl": 11.331315994262695,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -31437489.046728972,
|
|
"logits/rejected": -34931175.924764894,
|
|
"logps/chosen": -467.008128894081,
|
|
"logps/rejected": -386.536417515674,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.2069878578186035,
|
|
"metrics/advantage_var": 402.5113220214844,
|
|
"regularization/lipschitz_norm": 1330.2119140625,
|
|
"regularization/mmd": 0.1936829388141632,
|
|
"regularization/rkhs_norm": 230.5749053955078,
|
|
"regularization/w1": 1.1173781156539917,
|
|
"rewards/chosen": 0.4889666120582652,
|
|
"rewards/margins": 0.8321314207561002,
|
|
"rewards/rejected": -0.34316480869783506,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 17.722761154174805,
|
|
"kl": 11.810046195983887,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -33378682.041733548,
|
|
"logits/rejected": -35160207.39117199,
|
|
"logps/chosen": -475.239165329053,
|
|
"logps/rejected": -384.67006754185695,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.2113449573516846,
|
|
"metrics/advantage_var": 413.6997985839844,
|
|
"regularization/lipschitz_norm": 1309.820556640625,
|
|
"regularization/mmd": 0.19192849099636078,
|
|
"regularization/rkhs_norm": 228.4862823486328,
|
|
"regularization/w1": 1.1002492904663086,
|
|
"rewards/chosen": 0.45422147747792935,
|
|
"rewards/margins": 0.8251634756853352,
|
|
"rewards/rejected": -0.3709419982074058,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 14.714356422424316,
|
|
"kl": 8.73776912689209,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34605199.56509299,
|
|
"logits/rejected": -35368466.5060241,
|
|
"logps/chosen": -482.5021012160229,
|
|
"logps/rejected": -389.9052818416523,
|
|
"loss": 0.5641,
|
|
"loss/base_kto": 3.1989588737487793,
|
|
"metrics/advantage_var": 478.6230163574219,
|
|
"regularization/lipschitz_norm": 1323.5806884765625,
|
|
"regularization/mmd": 0.20208410918712616,
|
|
"regularization/rkhs_norm": 240.57632446289062,
|
|
"regularization/w1": 1.1118078231811523,
|
|
"rewards/chosen": 0.44558039107206726,
|
|
"rewards/margins": 0.8767346257563401,
|
|
"rewards/rejected": -0.43115423468427283,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 18.295270919799805,
|
|
"kl": 13.62393856048584,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34508696.935064934,
|
|
"logits/rejected": -36238909.686746985,
|
|
"logps/chosen": -530.406047077922,
|
|
"logps/rejected": -362.4706325301205,
|
|
"loss": 0.5539,
|
|
"loss/base_kto": 3.1301259994506836,
|
|
"metrics/advantage_var": 445.87139892578125,
|
|
"regularization/lipschitz_norm": 1313.7607421875,
|
|
"regularization/mmd": 0.19779208302497864,
|
|
"regularization/rkhs_norm": 235.46678161621094,
|
|
"regularization/w1": 1.1035590171813965,
|
|
"rewards/chosen": 0.5494497720297281,
|
|
"rewards/margins": 0.9158441597000382,
|
|
"rewards/rejected": -0.36639438767031013,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 17.067087173461914,
|
|
"kl": 12.12473201751709,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -32465490.372239746,
|
|
"logits/rejected": -34711282.526315786,
|
|
"logps/chosen": -473.06111987381706,
|
|
"logps/rejected": -357.53052438080493,
|
|
"loss": 0.5545,
|
|
"loss/base_kto": 3.1900439262390137,
|
|
"metrics/advantage_var": 419.9374084472656,
|
|
"regularization/lipschitz_norm": 1256.192626953125,
|
|
"regularization/mmd": 0.19065237045288086,
|
|
"regularization/rkhs_norm": 226.9671173095703,
|
|
"regularization/w1": 1.055201768875122,
|
|
"rewards/chosen": 0.512193649725207,
|
|
"rewards/margins": 0.8468726775542148,
|
|
"rewards/rejected": -0.3346790278290078,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 20.583120346069336,
|
|
"kl": 12.695433616638184,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34495723.56807512,
|
|
"logits/rejected": -34164614.58970359,
|
|
"logps/chosen": -461.06875978090767,
|
|
"logps/rejected": -383.80703978159124,
|
|
"loss": 0.5507,
|
|
"loss/base_kto": 3.2116639614105225,
|
|
"metrics/advantage_var": 362.9190673828125,
|
|
"regularization/lipschitz_norm": 1205.908447265625,
|
|
"regularization/mmd": 0.18123646080493927,
|
|
"regularization/rkhs_norm": 215.7576904296875,
|
|
"regularization/w1": 1.0129631757736206,
|
|
"rewards/chosen": 0.4543029594122897,
|
|
"rewards/margins": 0.8202510458220742,
|
|
"rewards/rejected": -0.3659480864097845,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 16.20465660095215,
|
|
"kl": 12.763381958007812,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34506081.689291105,
|
|
"logits/rejected": -35597977.51701783,
|
|
"logps/chosen": -472.95512820512823,
|
|
"logps/rejected": -369.6152248784441,
|
|
"loss": 0.5539,
|
|
"loss/base_kto": 3.140606641769409,
|
|
"metrics/advantage_var": 419.79345703125,
|
|
"regularization/lipschitz_norm": 1304.9725341796875,
|
|
"regularization/mmd": 0.1942991465330124,
|
|
"regularization/rkhs_norm": 231.30850219726562,
|
|
"regularization/w1": 1.0961769819259644,
|
|
"rewards/chosen": 0.5681098489200368,
|
|
"rewards/margins": 0.9044474655298296,
|
|
"rewards/rejected": -0.33633761660979283,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 16.107254028320312,
|
|
"kl": 14.743173599243164,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -33537325.17647059,
|
|
"logits/rejected": -35016920.42902208,
|
|
"logps/chosen": -491.8492647058824,
|
|
"logps/rejected": -352.57445287854887,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.1763598918914795,
|
|
"metrics/advantage_var": 435.5909118652344,
|
|
"regularization/lipschitz_norm": 1305.9234619140625,
|
|
"regularization/mmd": 0.19652919471263885,
|
|
"regularization/rkhs_norm": 233.9633331298828,
|
|
"regularization/w1": 1.0969756841659546,
|
|
"rewards/chosen": 0.5630381203288264,
|
|
"rewards/margins": 0.8732626353592631,
|
|
"rewards/rejected": -0.3102245150304367,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 21.481182098388672,
|
|
"kl": 12.255326271057129,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34864933.22255193,
|
|
"logits/rejected": -34594741.650165014,
|
|
"logps/chosen": -468.8024851632047,
|
|
"logps/rejected": -391.146349009901,
|
|
"loss": 0.5618,
|
|
"loss/base_kto": 3.2057454586029053,
|
|
"metrics/advantage_var": 422.9691467285156,
|
|
"regularization/lipschitz_norm": 1303.2862548828125,
|
|
"regularization/mmd": 0.194209486246109,
|
|
"regularization/rkhs_norm": 231.20176696777344,
|
|
"regularization/w1": 1.0947604179382324,
|
|
"rewards/chosen": 0.5268299317855156,
|
|
"rewards/margins": 0.8234247592675314,
|
|
"rewards/rejected": -0.29659482748201577,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 14.624382972717285,
|
|
"kl": 17.491533279418945,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34489799.84516129,
|
|
"logits/rejected": -35485540.84848485,
|
|
"logps/chosen": -480.8692540322581,
|
|
"logps/rejected": -394.1696732954546,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.1768290996551514,
|
|
"metrics/advantage_var": 424.4175720214844,
|
|
"regularization/lipschitz_norm": 1324.0650634765625,
|
|
"regularization/mmd": 0.1969681978225708,
|
|
"regularization/rkhs_norm": 234.4859619140625,
|
|
"regularization/w1": 1.1122145652770996,
|
|
"rewards/chosen": 0.5332314768145161,
|
|
"rewards/margins": 0.8496638054721866,
|
|
"rewards/rejected": -0.31643232865767046,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 13.645767211914062,
|
|
"kl": 13.762918472290039,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34331302.43962848,
|
|
"logits/rejected": -35779464.479495265,
|
|
"logps/chosen": -468.5385061919505,
|
|
"logps/rejected": -383.44178824921136,
|
|
"loss": 0.5633,
|
|
"loss/base_kto": 3.2114410400390625,
|
|
"metrics/advantage_var": 440.93072509765625,
|
|
"regularization/lipschitz_norm": 1309.8765869140625,
|
|
"regularization/mmd": 0.1943253129720688,
|
|
"regularization/rkhs_norm": 231.33963012695312,
|
|
"regularization/w1": 1.1002962589263916,
|
|
"rewards/chosen": 0.5259264305280089,
|
|
"rewards/margins": 0.8284625856751402,
|
|
"rewards/rejected": -0.30253615514713134,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 15.173432350158691,
|
|
"kl": 14.218057632446289,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -32425249.391304348,
|
|
"logits/rejected": -34971718.8427673,
|
|
"logps/chosen": -503.3925659937888,
|
|
"logps/rejected": -353.57033706761007,
|
|
"loss": 0.5588,
|
|
"loss/base_kto": 3.1940276622772217,
|
|
"metrics/advantage_var": 400.8150329589844,
|
|
"regularization/lipschitz_norm": 1293.98583984375,
|
|
"regularization/mmd": 0.18915407359600067,
|
|
"regularization/rkhs_norm": 225.1834259033203,
|
|
"regularization/w1": 1.086948037147522,
|
|
"rewards/chosen": 0.5563917456206328,
|
|
"rewards/margins": 0.836030096005423,
|
|
"rewards/rejected": -0.2796383503847902,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 14.659303665161133,
|
|
"kl": 14.139813423156738,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -33578268.62619808,
|
|
"logits/rejected": -36143078.94801223,
|
|
"logps/chosen": -490.8719049520767,
|
|
"logps/rejected": -378.82160263761466,
|
|
"loss": 0.5556,
|
|
"loss/base_kto": 3.136568546295166,
|
|
"metrics/advantage_var": 440.8300476074219,
|
|
"regularization/lipschitz_norm": 1323.760009765625,
|
|
"regularization/mmd": 0.19664309918880463,
|
|
"regularization/rkhs_norm": 234.0989227294922,
|
|
"regularization/w1": 1.111958384513855,
|
|
"rewards/chosen": 0.5801238526170627,
|
|
"rewards/margins": 0.9139969364491298,
|
|
"rewards/rejected": -0.3338730838320671,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.051947593688965,
|
|
"kl": 17.49016761779785,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -33601833.18611988,
|
|
"logits/rejected": -35005481.213622294,
|
|
"logps/chosen": -489.55293769716087,
|
|
"logps/rejected": -371.22288602941177,
|
|
"loss": 0.5558,
|
|
"loss/base_kto": 3.148218870162964,
|
|
"metrics/advantage_var": 409.5477600097656,
|
|
"regularization/lipschitz_norm": 1313.3740234375,
|
|
"regularization/mmd": 0.19498609006404877,
|
|
"regularization/rkhs_norm": 232.1262969970703,
|
|
"regularization/w1": 1.1032341718673706,
|
|
"rewards/chosen": 0.5496336157765674,
|
|
"rewards/margins": 0.8620950200159976,
|
|
"rewards/rejected": -0.31246140423943014,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 18.560632705688477,
|
|
"kl": 18.206375122070312,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -33359756.2003231,
|
|
"logits/rejected": -34790556.46596067,
|
|
"logps/chosen": -497.78165387722134,
|
|
"logps/rejected": -370.53711232980334,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.1513259410858154,
|
|
"metrics/advantage_var": 684.828857421875,
|
|
"regularization/lipschitz_norm": 1390.2115478515625,
|
|
"regularization/mmd": 0.20519761741161346,
|
|
"regularization/rkhs_norm": 244.2828826904297,
|
|
"regularization/w1": 1.16777765750885,
|
|
"rewards/chosen": 0.6012356315945577,
|
|
"rewards/margins": 0.8369443097780629,
|
|
"rewards/rejected": -0.2357086781835051,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 14.579899787902832,
|
|
"kl": 15.151265144348145,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34251067.59344263,
|
|
"logits/rejected": -36550078.280597016,
|
|
"logps/chosen": -491.97843237704916,
|
|
"logps/rejected": -378.91923973880597,
|
|
"loss": 0.5523,
|
|
"loss/base_kto": 3.1356801986694336,
|
|
"metrics/advantage_var": 447.5450134277344,
|
|
"regularization/lipschitz_norm": 1292.7020263671875,
|
|
"regularization/mmd": 0.19711998105049133,
|
|
"regularization/rkhs_norm": 234.66665649414062,
|
|
"regularization/w1": 1.0858697891235352,
|
|
"rewards/chosen": 0.585163354091957,
|
|
"rewards/margins": 0.8870121271942191,
|
|
"rewards/rejected": -0.3018487731022621,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 17.755054473876953,
|
|
"kl": 14.024542808532715,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34082644.508856684,
|
|
"logits/rejected": -36033904.26707132,
|
|
"logps/chosen": -469.70632045088564,
|
|
"logps/rejected": -371.58156297420334,
|
|
"loss": 0.5474,
|
|
"loss/base_kto": 3.205434560775757,
|
|
"metrics/advantage_var": 382.1590270996094,
|
|
"regularization/lipschitz_norm": 1180.8150634765625,
|
|
"regularization/mmd": 0.18211029469966888,
|
|
"regularization/rkhs_norm": 216.79795837402344,
|
|
"regularization/w1": 0.9918846487998962,
|
|
"rewards/chosen": 0.5159232958119464,
|
|
"rewards/margins": 0.8224332833492899,
|
|
"rewards/rejected": -0.3065099875373435,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 16.099609375,
|
|
"kl": 16.29318618774414,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34512733.717908084,
|
|
"logits/rejected": -35207549.83050848,
|
|
"logps/chosen": -467.43304278922346,
|
|
"logps/rejected": -385.44551714175657,
|
|
"loss": 0.556,
|
|
"loss/base_kto": 3.1834332942962646,
|
|
"metrics/advantage_var": 420.833251953125,
|
|
"regularization/lipschitz_norm": 1275.8087158203125,
|
|
"regularization/mmd": 0.1927143633365631,
|
|
"regularization/rkhs_norm": 229.42185974121094,
|
|
"regularization/w1": 1.0716793537139893,
|
|
"rewards/chosen": 0.5675009169782587,
|
|
"rewards/margins": 0.8446744221529866,
|
|
"rewards/rejected": -0.27717350517472794,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 22.56321907043457,
|
|
"kl": 15.587442398071289,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34523900.67532468,
|
|
"logits/rejected": -36209404.915662654,
|
|
"logps/chosen": -493.51861810064935,
|
|
"logps/rejected": -401.04593373493975,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.170428991317749,
|
|
"metrics/advantage_var": 408.4739685058594,
|
|
"regularization/lipschitz_norm": 1338.758056640625,
|
|
"regularization/mmd": 0.19401060044765472,
|
|
"regularization/rkhs_norm": 230.96499633789062,
|
|
"regularization/w1": 1.1245567798614502,
|
|
"rewards/chosen": 0.5469281085125812,
|
|
"rewards/margins": 0.8478588670692688,
|
|
"rewards/rejected": -0.30093075855668766,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 17.010610580444336,
|
|
"kl": 13.440454483032227,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33436187.28526646,
|
|
"logits/rejected": -35349768.77258567,
|
|
"logps/chosen": -464.2762539184953,
|
|
"logps/rejected": -364.92752141744546,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.212057590484619,
|
|
"metrics/advantage_var": 375.243408203125,
|
|
"regularization/lipschitz_norm": 1230.5667724609375,
|
|
"regularization/mmd": 0.18288172781467438,
|
|
"regularization/rkhs_norm": 217.71633911132812,
|
|
"regularization/w1": 1.033676028251648,
|
|
"rewards/chosen": 0.49261713775347765,
|
|
"rewards/margins": 0.7861720914504208,
|
|
"rewards/rejected": -0.2935549536969431,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 17.789718627929688,
|
|
"kl": 13.67638874053955,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34302437.80465116,
|
|
"logits/rejected": -35579897.5496063,
|
|
"logps/chosen": -493.7002906976744,
|
|
"logps/rejected": -393.74451279527557,
|
|
"loss": 0.571,
|
|
"loss/base_kto": 3.2242684364318848,
|
|
"metrics/advantage_var": 441.3744812011719,
|
|
"regularization/lipschitz_norm": 1364.3621826171875,
|
|
"regularization/mmd": 0.19798366725444794,
|
|
"regularization/rkhs_norm": 235.69485473632812,
|
|
"regularization/w1": 1.1460641622543335,
|
|
"rewards/chosen": 0.5166667612948159,
|
|
"rewards/margins": 0.8272236806630492,
|
|
"rewards/rejected": -0.31055691936823326,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 12.054067611694336,
|
|
"kl": 12.401045799255371,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33550323.569044005,
|
|
"logits/rejected": -33446841.095008053,
|
|
"logps/chosen": -470.05202010622156,
|
|
"logps/rejected": -343.1029589371981,
|
|
"loss": 0.548,
|
|
"loss/base_kto": 3.1565399169921875,
|
|
"metrics/advantage_var": 404.8399963378906,
|
|
"regularization/lipschitz_norm": 1240.16796875,
|
|
"regularization/mmd": 0.18549861013889313,
|
|
"regularization/rkhs_norm": 220.83169555664062,
|
|
"regularization/w1": 1.0417410135269165,
|
|
"rewards/chosen": 0.5369251118082559,
|
|
"rewards/margins": 0.8793018207926309,
|
|
"rewards/rejected": -0.342376708984375,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_loss": 0.5871853746074881,
|
|
"train_runtime": 11079.3298,
|
|
"train_samples_per_second": 13.378,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|