8bcf662309
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed2024 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 24.32085418701172,
|
|
"kl": 6.620049953460693,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35861141.06329114,
|
|
"logits/rejected": -37501882.469135806,
|
|
"logps/chosen": -501.94417523734177,
|
|
"logps/rejected": -375.9259741512346,
|
|
"loss": 0.5316,
|
|
"loss/base_kto": 3.8595731258392334,
|
|
"loss/total_with_kl": 4.253028869628906,
|
|
"metrics/advantage_var": 236.3098907470703,
|
|
"regularization/advantage_var": 236.3098907470703,
|
|
"regularization/kl": 0.39345595240592957,
|
|
"rewards/chosen": 0.02552956569043896,
|
|
"rewards/margins": 0.14411608236509293,
|
|
"rewards/rejected": -0.11858651667465399,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 12.490849494934082,
|
|
"kl": 26.332868576049805,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35957793.13405239,
|
|
"logits/rejected": -38035605.29952456,
|
|
"logps/chosen": -489.97972842835134,
|
|
"logps/rejected": -359.01057349445324,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.813135862350464,
|
|
"loss/total_with_kl": 4.144712924957275,
|
|
"metrics/advantage_var": 199.14524841308594,
|
|
"regularization/advantage_var": 199.14524841308594,
|
|
"regularization/kl": 0.3315768241882324,
|
|
"rewards/chosen": 0.4595373591583205,
|
|
"rewards/margins": 0.16364866697051095,
|
|
"rewards/rejected": 0.29588869218780955,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 16.590961456298828,
|
|
"kl": 3.329580068588257,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35121857.64102564,
|
|
"logits/rejected": -36009749.853658535,
|
|
"logps/chosen": -470.4701522435897,
|
|
"logps/rejected": -365.0965129573171,
|
|
"loss": 0.5163,
|
|
"loss/base_kto": 3.8311352729797363,
|
|
"loss/total_with_kl": 4.130624294281006,
|
|
"metrics/advantage_var": 179.87359619140625,
|
|
"regularization/advantage_var": 179.87359619140625,
|
|
"regularization/kl": 0.2994895279407501,
|
|
"rewards/chosen": 0.015784152043171417,
|
|
"rewards/margins": 0.15219200421602297,
|
|
"rewards/rejected": -0.13640785217285156,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 15.667641639709473,
|
|
"kl": 12.654009819030762,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37766268.17007874,
|
|
"logits/rejected": -38015642.79069767,
|
|
"logps/chosen": -504.93292322834645,
|
|
"logps/rejected": -384.18226744186046,
|
|
"loss": 0.5236,
|
|
"loss/base_kto": 3.8299171924591064,
|
|
"loss/total_with_kl": 4.188797473907471,
|
|
"metrics/advantage_var": 215.5437774658203,
|
|
"regularization/advantage_var": 215.5437774658203,
|
|
"regularization/kl": 0.3588803708553314,
|
|
"rewards/chosen": 0.2038393035648376,
|
|
"rewards/margins": 0.16062911575634983,
|
|
"rewards/rejected": 0.04321018780848777,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 14.052574157714844,
|
|
"kl": 4.885131359100342,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36457730.74425727,
|
|
"logits/rejected": -37892599.01754386,
|
|
"logps/chosen": -491.5156489280245,
|
|
"logps/rejected": -374.7734250398724,
|
|
"loss": 0.5269,
|
|
"loss/base_kto": 3.8030097484588623,
|
|
"loss/total_with_kl": 4.215572357177734,
|
|
"metrics/advantage_var": 247.7853240966797,
|
|
"regularization/advantage_var": 247.7853240966797,
|
|
"regularization/kl": 0.41256260871887207,
|
|
"rewards/chosen": -0.02621486833230642,
|
|
"rewards/margins": 0.2226151938281551,
|
|
"rewards/rejected": -0.24883006216046152,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 16.11071014404297,
|
|
"kl": 18.43494415283203,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -37538017.123664126,
|
|
"logits/rejected": -37439799.296,
|
|
"logps/chosen": -490.54790076335877,
|
|
"logps/rejected": -360.4264,
|
|
"loss": 0.515,
|
|
"loss/base_kto": 3.7947540283203125,
|
|
"loss/total_with_kl": 4.120279312133789,
|
|
"metrics/advantage_var": 195.51060485839844,
|
|
"regularization/advantage_var": 195.51060485839844,
|
|
"regularization/kl": 0.3255251348018646,
|
|
"rewards/chosen": 0.34407502385496186,
|
|
"rewards/margins": 0.18177883244871185,
|
|
"rewards/rejected": 0.16229619140625,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 13.922213554382324,
|
|
"kl": 11.039660453796387,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35708098.74846626,
|
|
"logits/rejected": -38608390.522292994,
|
|
"logps/chosen": -484.8937404141104,
|
|
"logps/rejected": -383.91650079617835,
|
|
"loss": 0.5165,
|
|
"loss/base_kto": 3.730560302734375,
|
|
"loss/total_with_kl": 4.132201671600342,
|
|
"metrics/advantage_var": 241.22573852539062,
|
|
"regularization/advantage_var": 241.22573852539062,
|
|
"regularization/kl": 0.4016408622264862,
|
|
"rewards/chosen": 0.25796843452687646,
|
|
"rewards/margins": 0.23204063272649486,
|
|
"rewards/rejected": 0.025927801800381605,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 15.479750633239746,
|
|
"kl": 24.18060874938965,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -37649381.90104947,
|
|
"logits/rejected": -37157214.799347475,
|
|
"logps/chosen": -493.6107102698651,
|
|
"logps/rejected": -372.21951468189235,
|
|
"loss": 0.5144,
|
|
"loss/base_kto": 3.7350997924804688,
|
|
"loss/total_with_kl": 4.114933967590332,
|
|
"metrics/advantage_var": 228.1287078857422,
|
|
"regularization/advantage_var": 228.1287078857422,
|
|
"regularization/kl": 0.3798343241214752,
|
|
"rewards/chosen": 0.4730671620976621,
|
|
"rewards/margins": 0.2011343828135318,
|
|
"rewards/rejected": 0.2719327792841303,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 18.767160415649414,
|
|
"kl": 36.3804817199707,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36200173.30551416,
|
|
"logits/rejected": -37414057.82594417,
|
|
"logps/chosen": -463.64535208643815,
|
|
"logps/rejected": -356.4462746305419,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.7325439453125,
|
|
"loss/total_with_kl": 4.099050521850586,
|
|
"metrics/advantage_var": 220.1240692138672,
|
|
"regularization/advantage_var": 220.1240692138672,
|
|
"regularization/kl": 0.3665066361427307,
|
|
"rewards/chosen": 0.6073920396330105,
|
|
"rewards/margins": 0.22740391893854467,
|
|
"rewards/rejected": 0.3799881206944658,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 11.879533767700195,
|
|
"kl": 32.49602127075195,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37876213.36498516,
|
|
"logits/rejected": -39788665.66336634,
|
|
"logps/chosen": -486.05471068249255,
|
|
"logps/rejected": -380.03282281353137,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.7456014156341553,
|
|
"loss/total_with_kl": 4.20007848739624,
|
|
"metrics/advantage_var": 272.9594421386719,
|
|
"regularization/advantage_var": 272.9594421386719,
|
|
"regularization/kl": 0.4544773995876312,
|
|
"rewards/chosen": 0.5189340744244946,
|
|
"rewards/margins": 0.19561730518653564,
|
|
"rewards/rejected": 0.32331676923795893,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.6059513092041,
|
|
"kl": 30.7705078125,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37962922.9544688,
|
|
"logits/rejected": -38796179.19068413,
|
|
"logps/chosen": -475.97112141652616,
|
|
"logps/rejected": -378.00206968704515,
|
|
"loss": 0.5136,
|
|
"loss/base_kto": 3.6783127784729004,
|
|
"loss/total_with_kl": 4.108422756195068,
|
|
"metrics/advantage_var": 258.3245544433594,
|
|
"regularization/advantage_var": 258.3245544433594,
|
|
"regularization/kl": 0.430110365152359,
|
|
"rewards/chosen": 0.48481614111245785,
|
|
"rewards/margins": 0.2808650342705501,
|
|
"rewards/rejected": 0.20395110684190776,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 13.229418754577637,
|
|
"kl": 13.234227180480957,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36810953.22979398,
|
|
"logits/rejected": -37116424.6779661,
|
|
"logps/chosen": -465.34152139461173,
|
|
"logps/rejected": -377.6193663328197,
|
|
"loss": 0.5197,
|
|
"loss/base_kto": 3.7048676013946533,
|
|
"loss/total_with_kl": 4.157932758331299,
|
|
"metrics/advantage_var": 272.11114501953125,
|
|
"regularization/advantage_var": 272.11114501953125,
|
|
"regularization/kl": 0.45306500792503357,
|
|
"rewards/chosen": 0.14031385128547197,
|
|
"rewards/margins": 0.3018961750151884,
|
|
"rewards/rejected": -0.1615823237297164,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 20.17181968688965,
|
|
"kl": 11.351752281188965,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36860006.080996886,
|
|
"logits/rejected": -36849118.29467085,
|
|
"logps/chosen": -478.508761682243,
|
|
"logps/rejected": -378.24250587774293,
|
|
"loss": 0.5292,
|
|
"loss/base_kto": 3.760735034942627,
|
|
"loss/total_with_kl": 4.233994960784912,
|
|
"metrics/advantage_var": 284.2402038574219,
|
|
"regularization/advantage_var": 284.2402038574219,
|
|
"regularization/kl": 0.473259836435318,
|
|
"rewards/chosen": 0.1905547168767341,
|
|
"rewards/margins": 0.20015269176158046,
|
|
"rewards/rejected": -0.009597974884846367,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 14.432672500610352,
|
|
"kl": 12.176385879516602,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36651500.97576737,
|
|
"logits/rejected": -39384570.577912256,
|
|
"logps/chosen": -504.3191639741519,
|
|
"logps/rejected": -364.26059001512857,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.6998088359832764,
|
|
"loss/total_with_kl": 4.121967315673828,
|
|
"metrics/advantage_var": 253.5484161376953,
|
|
"regularization/advantage_var": 253.5484161376953,
|
|
"regularization/kl": 0.4221580922603607,
|
|
"rewards/chosen": 0.2182309546647819,
|
|
"rewards/margins": 0.27326805060466053,
|
|
"rewards/rejected": -0.055037095939878614,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 14.449652671813965,
|
|
"kl": 8.657655715942383,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36689699.25748503,
|
|
"logits/rejected": -38683975.94771242,
|
|
"logps/chosen": -462.6022642215569,
|
|
"logps/rejected": -374.1436376633987,
|
|
"loss": 0.5214,
|
|
"loss/base_kto": 3.7633156776428223,
|
|
"loss/total_with_kl": 4.171494483947754,
|
|
"metrics/advantage_var": 245.15245056152344,
|
|
"regularization/advantage_var": 245.15245056152344,
|
|
"regularization/kl": 0.40817880630493164,
|
|
"rewards/chosen": 0.12144154988363118,
|
|
"rewards/margins": 0.23151773113331203,
|
|
"rewards/rejected": -0.11007618124968087,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 13.811630249023438,
|
|
"kl": 21.14801597595215,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -38208566.59127625,
|
|
"logits/rejected": -38600158.692889564,
|
|
"logps/chosen": -520.9845516962844,
|
|
"logps/rejected": -393.59805219364597,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.6663787364959717,
|
|
"loss/total_with_kl": 4.121258735656738,
|
|
"metrics/advantage_var": 273.2014465332031,
|
|
"regularization/advantage_var": 273.2014465332031,
|
|
"regularization/kl": 0.45488038659095764,
|
|
"rewards/chosen": 0.4207693170846375,
|
|
"rewards/margins": 0.2904753714027871,
|
|
"rewards/rejected": 0.1302939456818504,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.25345802307129,
|
|
"kl": 4.230582237243652,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -34972668.7284345,
|
|
"logits/rejected": -36581563.889908254,
|
|
"logps/chosen": -471.2091154153355,
|
|
"logps/rejected": -365.0969036697248,
|
|
"loss": 0.5219,
|
|
"loss/base_kto": 3.714958906173706,
|
|
"loss/total_with_kl": 4.175113201141357,
|
|
"metrics/advantage_var": 276.3685607910156,
|
|
"regularization/advantage_var": 276.3685607910156,
|
|
"regularization/kl": 0.4601536691188812,
|
|
"rewards/chosen": 0.03428346432816868,
|
|
"rewards/margins": 0.2866421838068575,
|
|
"rewards/rejected": -0.2523587194786888,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 13.907388687133789,
|
|
"kl": 2.6898460388183594,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34957206.126009695,
|
|
"logits/rejected": -36393083.93343419,
|
|
"logps/chosen": -497.07017366720515,
|
|
"logps/rejected": -366.5627600226929,
|
|
"loss": 0.508,
|
|
"loss/base_kto": 3.656599760055542,
|
|
"loss/total_with_kl": 4.064292907714844,
|
|
"metrics/advantage_var": 244.8606414794922,
|
|
"regularization/advantage_var": 244.8606414794922,
|
|
"regularization/kl": 0.40769296884536743,
|
|
"rewards/chosen": -0.03925414077684837,
|
|
"rewards/margins": 0.337245006393041,
|
|
"rewards/rejected": -0.37649914716988936,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 13.988927841186523,
|
|
"kl": 2.5878779888153076,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36559256.31397174,
|
|
"logits/rejected": -36681259.79471228,
|
|
"logps/chosen": -482.8315345368917,
|
|
"logps/rejected": -356.6305647356143,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.727613925933838,
|
|
"loss/total_with_kl": 4.237107753753662,
|
|
"metrics/advantage_var": 306.0021057128906,
|
|
"regularization/advantage_var": 306.0021057128906,
|
|
"regularization/kl": 0.5094935297966003,
|
|
"rewards/chosen": -0.0574733788027681,
|
|
"rewards/margins": 0.26809830134035983,
|
|
"rewards/rejected": -0.32557168014312793,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 12.955965995788574,
|
|
"kl": 3.9791793823242188,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36830659.86031746,
|
|
"logits/rejected": -36714439.286153845,
|
|
"logps/chosen": -482.76468253968255,
|
|
"logps/rejected": -390.1410576923077,
|
|
"loss": 0.5134,
|
|
"loss/base_kto": 3.6922378540039062,
|
|
"loss/total_with_kl": 4.107511520385742,
|
|
"metrics/advantage_var": 249.4137420654297,
|
|
"regularization/advantage_var": 249.4137420654297,
|
|
"regularization/kl": 0.41527387499809265,
|
|
"rewards/chosen": -0.03859777601938399,
|
|
"rewards/margins": 0.3019872929033925,
|
|
"rewards/rejected": -0.34058506892277646,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 11.858966827392578,
|
|
"kl": 3.107635259628296,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37624990.283911675,
|
|
"logits/rejected": -38347227.54179566,
|
|
"logps/chosen": -479.9210370662461,
|
|
"logps/rejected": -360.61368034055727,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.744265079498291,
|
|
"loss/total_with_kl": 4.1571197509765625,
|
|
"metrics/advantage_var": 247.960693359375,
|
|
"regularization/advantage_var": 247.960693359375,
|
|
"regularization/kl": 0.4128545820713043,
|
|
"rewards/chosen": -0.07257702222757911,
|
|
"rewards/margins": 0.239034141499322,
|
|
"rewards/rejected": -0.3116111637269011,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 12.195089340209961,
|
|
"kl": 4.067375183105469,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35495411.358024694,
|
|
"logits/rejected": -36405591.49367089,
|
|
"logps/chosen": -455.54841820987656,
|
|
"logps/rejected": -378.18814280063293,
|
|
"loss": 0.5203,
|
|
"loss/base_kto": 3.71710205078125,
|
|
"loss/total_with_kl": 4.162749767303467,
|
|
"metrics/advantage_var": 267.65625,
|
|
"regularization/advantage_var": 267.65625,
|
|
"regularization/kl": 0.44564762711524963,
|
|
"rewards/chosen": 0.07983982415846837,
|
|
"rewards/margins": 0.2812370198055028,
|
|
"rewards/rejected": -0.20139719564703445,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 15.630626678466797,
|
|
"kl": 10.41035270690918,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35388355.029007636,
|
|
"logits/rejected": -35857553.8176,
|
|
"logps/chosen": -465.7159828244275,
|
|
"logps/rejected": -357.2461,
|
|
"loss": 0.5108,
|
|
"loss/base_kto": 3.6266372203826904,
|
|
"loss/total_with_kl": 4.086459636688232,
|
|
"metrics/advantage_var": 276.1694030761719,
|
|
"regularization/advantage_var": 276.1694030761719,
|
|
"regularization/kl": 0.4598220884799957,
|
|
"rewards/chosen": 0.22740483174797232,
|
|
"rewards/margins": 0.3562315895604723,
|
|
"rewards/rejected": -0.1288267578125,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 15.51076602935791,
|
|
"kl": 20.242170333862305,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36996972.14132105,
|
|
"logits/rejected": -37566604.0063593,
|
|
"logps/chosen": -493.4741743471582,
|
|
"logps/rejected": -387.02091613672496,
|
|
"loss": 0.5283,
|
|
"loss/base_kto": 3.663759708404541,
|
|
"loss/total_with_kl": 4.226189136505127,
|
|
"metrics/advantage_var": 337.7952575683594,
|
|
"regularization/advantage_var": 337.7952575683594,
|
|
"regularization/kl": 0.5624291300773621,
|
|
"rewards/chosen": 0.4211090134768625,
|
|
"rewards/margins": 0.3138429770475104,
|
|
"rewards/rejected": 0.10726603642935215,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.41870880126953,
|
|
"kl": 27.8725528717041,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35428084.59600615,
|
|
"logits/rejected": -37382629.13831478,
|
|
"logps/chosen": -462.9358198924731,
|
|
"logps/rejected": -382.7552662957075,
|
|
"loss": 0.5079,
|
|
"loss/base_kto": 3.6638245582580566,
|
|
"loss/total_with_kl": 4.063495635986328,
|
|
"metrics/advantage_var": 240.042724609375,
|
|
"regularization/advantage_var": 240.042724609375,
|
|
"regularization/kl": 0.39967110753059387,
|
|
"rewards/chosen": 0.4316009662118376,
|
|
"rewards/margins": 0.24892917213921487,
|
|
"rewards/rejected": 0.1826717940726227,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 15.007033348083496,
|
|
"kl": 13.093035697937012,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36440083.38170347,
|
|
"logits/rejected": -37480203.888544895,
|
|
"logps/chosen": -464.44829455835963,
|
|
"logps/rejected": -378.70474554953563,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.7181496620178223,
|
|
"loss/total_with_kl": 4.156769752502441,
|
|
"metrics/advantage_var": 263.4352722167969,
|
|
"regularization/advantage_var": 263.4352722167969,
|
|
"regularization/kl": 0.4386197030544281,
|
|
"rewards/chosen": 0.21653328784256703,
|
|
"rewards/margins": 0.24704505239255464,
|
|
"rewards/rejected": -0.030511764549987604,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 21.0418701171875,
|
|
"kl": 16.60923957824707,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36879919.537290715,
|
|
"logits/rejected": -36814105.06581059,
|
|
"logps/chosen": -499.548848934551,
|
|
"logps/rejected": -359.74197431781704,
|
|
"loss": 0.5134,
|
|
"loss/base_kto": 3.6128647327423096,
|
|
"loss/total_with_kl": 4.107340335845947,
|
|
"metrics/advantage_var": 296.9823913574219,
|
|
"regularization/advantage_var": 296.9823913574219,
|
|
"regularization/kl": 0.49447569251060486,
|
|
"rewards/chosen": 0.3630597072467775,
|
|
"rewards/margins": 0.37933509169968305,
|
|
"rewards/rejected": -0.016275384452905547,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 20.375349044799805,
|
|
"kl": 25.070850372314453,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36709987.921331316,
|
|
"logits/rejected": -37055876.75605816,
|
|
"logps/chosen": -494.6037254160363,
|
|
"logps/rejected": -383.87136510500807,
|
|
"loss": 0.5273,
|
|
"loss/base_kto": 3.655336856842041,
|
|
"loss/total_with_kl": 4.21851110458374,
|
|
"metrics/advantage_var": 338.24273681640625,
|
|
"regularization/advantage_var": 338.24273681640625,
|
|
"regularization/kl": 0.5631741285324097,
|
|
"rewards/chosen": 0.4840269334016878,
|
|
"rewards/margins": 0.2985697623328646,
|
|
"rewards/rejected": 0.1854571710688232,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 11.905738830566406,
|
|
"kl": 26.71291160583496,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37440047.40740741,
|
|
"logits/rejected": -38528165.26582278,
|
|
"logps/chosen": -502.99652777777777,
|
|
"logps/rejected": -366.58603639240505,
|
|
"loss": 0.5069,
|
|
"loss/base_kto": 3.6144073009490967,
|
|
"loss/total_with_kl": 4.054908275604248,
|
|
"metrics/advantage_var": 264.5651550292969,
|
|
"regularization/advantage_var": 264.5651550292969,
|
|
"regularization/kl": 0.440500944852829,
|
|
"rewards/chosen": 0.5046551551347898,
|
|
"rewards/margins": 0.2937068629216395,
|
|
"rewards/rejected": 0.21094829221315023,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 15.570670127868652,
|
|
"kl": 15.21961498260498,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37499581.62962963,
|
|
"logits/rejected": -38416870.07594936,
|
|
"logps/chosen": -523.3862847222222,
|
|
"logps/rejected": -365.23763844936707,
|
|
"loss": 0.5129,
|
|
"loss/base_kto": 3.680920124053955,
|
|
"loss/total_with_kl": 4.1031928062438965,
|
|
"metrics/advantage_var": 253.6173858642578,
|
|
"regularization/advantage_var": 253.6173858642578,
|
|
"regularization/kl": 0.4222729206085205,
|
|
"rewards/chosen": 0.2762401722095631,
|
|
"rewards/margins": 0.2778101491218322,
|
|
"rewards/rejected": -0.0015699769122691095,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 18.33673667907715,
|
|
"kl": 13.349814414978027,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -34919093.41732284,
|
|
"logits/rejected": -37028608.39689922,
|
|
"logps/chosen": -475.5461614173228,
|
|
"logps/rejected": -383.7982315891473,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.6636300086975098,
|
|
"loss/total_with_kl": 4.178959369659424,
|
|
"metrics/advantage_var": 309.5070495605469,
|
|
"regularization/advantage_var": 309.5070495605469,
|
|
"regularization/kl": 0.5153291821479797,
|
|
"rewards/chosen": 0.21472533218503936,
|
|
"rewards/margins": 0.28177217926973536,
|
|
"rewards/rejected": -0.06704684708469598,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 17.798927307128906,
|
|
"kl": 10.555925369262695,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35837511.461412154,
|
|
"logits/rejected": -38261688.27421758,
|
|
"logps/chosen": -480.0981116584565,
|
|
"logps/rejected": -368.63720193740687,
|
|
"loss": 0.5162,
|
|
"loss/base_kto": 3.6703248023986816,
|
|
"loss/total_with_kl": 4.1295952796936035,
|
|
"metrics/advantage_var": 275.83819580078125,
|
|
"regularization/advantage_var": 275.83819580078125,
|
|
"regularization/kl": 0.45927056670188904,
|
|
"rewards/chosen": 0.162047525540557,
|
|
"rewards/margins": 0.29109999612725135,
|
|
"rewards/rejected": -0.1290524705866943,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 15.719472885131836,
|
|
"kl": 10.699854850769043,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -35092709.68224299,
|
|
"logits/rejected": -35829111.57366771,
|
|
"logps/chosen": -454.16856503115264,
|
|
"logps/rejected": -385.9563822492163,
|
|
"loss": 0.5147,
|
|
"loss/base_kto": 3.670102834701538,
|
|
"loss/total_with_kl": 4.117818355560303,
|
|
"metrics/advantage_var": 268.8981018066406,
|
|
"regularization/advantage_var": 268.8981018066406,
|
|
"regularization/kl": 0.4477153718471527,
|
|
"rewards/chosen": 0.16595170208226853,
|
|
"rewards/margins": 0.2956248616905415,
|
|
"rewards/rejected": -0.12967315960827291,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 16.269304275512695,
|
|
"kl": 19.776601791381836,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35707229.893416926,
|
|
"logits/rejected": -37422351.15264797,
|
|
"logps/chosen": -493.3255779780564,
|
|
"logps/rejected": -376.56371690031153,
|
|
"loss": 0.5158,
|
|
"loss/base_kto": 3.6863350868225098,
|
|
"loss/total_with_kl": 4.1266865730285645,
|
|
"metrics/advantage_var": 264.475341796875,
|
|
"regularization/advantage_var": 264.475341796875,
|
|
"regularization/kl": 0.4403514564037323,
|
|
"rewards/chosen": 0.36896426401168203,
|
|
"rewards/margins": 0.26900655129750756,
|
|
"rewards/rejected": 0.09995771271417446,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 12.744359016418457,
|
|
"kl": 11.336798667907715,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35560088.39055118,
|
|
"logits/rejected": -36831657.475968994,
|
|
"logps/chosen": -487.2536417322835,
|
|
"logps/rejected": -345.7433139534884,
|
|
"loss": 0.5088,
|
|
"loss/base_kto": 3.6728012561798096,
|
|
"loss/total_with_kl": 4.070383548736572,
|
|
"metrics/advantage_var": 238.7881622314453,
|
|
"regularization/advantage_var": 238.7881622314453,
|
|
"regularization/kl": 0.3975822627544403,
|
|
"rewards/chosen": 0.18450591320127951,
|
|
"rewards/margins": 0.2887475253621322,
|
|
"rewards/rejected": -0.10424161216085272,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 14.963236808776855,
|
|
"kl": 10.810988426208496,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36938595.313755795,
|
|
"logits/rejected": -38018664.341232225,
|
|
"logps/chosen": -500.0477202472952,
|
|
"logps/rejected": -379.4044727488152,
|
|
"loss": 0.5245,
|
|
"loss/base_kto": 3.689709424972534,
|
|
"loss/total_with_kl": 4.195741176605225,
|
|
"metrics/advantage_var": 303.9228210449219,
|
|
"regularization/advantage_var": 303.9228210449219,
|
|
"regularization/kl": 0.5060314536094666,
|
|
"rewards/chosen": 0.2027928678108699,
|
|
"rewards/margins": 0.28093195748260297,
|
|
"rewards/rejected": -0.07813908967173307,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.082501411437988,
|
|
"kl": 16.48919677734375,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -37628381.20388349,
|
|
"logits/rejected": -38292254.163141996,
|
|
"logps/chosen": -491.7256270226537,
|
|
"logps/rejected": -385.6554003021148,
|
|
"loss": 0.5114,
|
|
"loss/base_kto": 3.6569862365722656,
|
|
"loss/total_with_kl": 4.09153938293457,
|
|
"metrics/advantage_var": 260.99267578125,
|
|
"regularization/advantage_var": 260.99267578125,
|
|
"regularization/kl": 0.4345528185367584,
|
|
"rewards/chosen": 0.252661053802589,
|
|
"rewards/margins": 0.2780839135707218,
|
|
"rewards/rejected": -0.02542285976813279,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 12.365095138549805,
|
|
"kl": 11.425230979919434,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36415946.36190476,
|
|
"logits/rejected": -38076844.50461538,
|
|
"logps/chosen": -476.50565476190474,
|
|
"logps/rejected": -360.44697115384616,
|
|
"loss": 0.5075,
|
|
"loss/base_kto": 3.651010274887085,
|
|
"loss/total_with_kl": 4.060315132141113,
|
|
"metrics/advantage_var": 245.828369140625,
|
|
"regularization/advantage_var": 245.828369140625,
|
|
"regularization/kl": 0.4093042314052582,
|
|
"rewards/chosen": 0.12858787415519593,
|
|
"rewards/margins": 0.3083314795488257,
|
|
"rewards/rejected": -0.1797436053936298,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 9.462163925170898,
|
|
"kl": 7.94988489151001,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -35250398.03236246,
|
|
"logits/rejected": -35756571.92727273,
|
|
"logps/chosen": -466.686286407767,
|
|
"logps/rejected": -388.2438446969697,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.443466901779175,
|
|
"loss/total_with_kl": 4.199981212615967,
|
|
"metrics/advantage_var": 454.363037109375,
|
|
"regularization/advantage_var": 454.363037109375,
|
|
"regularization/kl": 0.7565144896507263,
|
|
"rewards/chosen": 0.2011941952998584,
|
|
"rewards/margins": 0.590251192888045,
|
|
"rewards/rejected": -0.38905699758818657,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 15.779438972473145,
|
|
"kl": 8.964886665344238,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36306812.275650844,
|
|
"logits/rejected": -37220629.63955343,
|
|
"logps/chosen": -463.8027852220521,
|
|
"logps/rejected": -387.6121162280702,
|
|
"loss": 0.5334,
|
|
"loss/base_kto": 3.242673635482788,
|
|
"loss/total_with_kl": 4.267380237579346,
|
|
"metrics/advantage_var": 615.4395751953125,
|
|
"regularization/advantage_var": 615.4395751953125,
|
|
"regularization/kl": 1.0247067213058472,
|
|
"rewards/chosen": 0.43634005162471284,
|
|
"rewards/margins": 0.843648992116701,
|
|
"rewards/rejected": -0.4073089404919881,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 16.526897430419922,
|
|
"kl": 18.56119155883789,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35259160.1509434,
|
|
"logits/rejected": -36981903.10559006,
|
|
"logps/chosen": -442.0420597484277,
|
|
"logps/rejected": -392.12567934782606,
|
|
"loss": 0.5062,
|
|
"loss/base_kto": 3.1583974361419678,
|
|
"loss/total_with_kl": 4.049962997436523,
|
|
"metrics/advantage_var": 535.4747924804688,
|
|
"regularization/advantage_var": 535.4747924804688,
|
|
"regularization/kl": 0.8915656208992004,
|
|
"rewards/chosen": 0.5975148902749116,
|
|
"rewards/margins": 0.886257548048895,
|
|
"rewards/rejected": -0.2887426577739834,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 13.164573669433594,
|
|
"kl": 5.3090996742248535,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35334127.29526917,
|
|
"logits/rejected": -35689172.629685156,
|
|
"logps/chosen": -512.6254078303426,
|
|
"logps/rejected": -366.82208114692656,
|
|
"loss": 0.5442,
|
|
"loss/base_kto": 3.19159197807312,
|
|
"loss/total_with_kl": 4.353483200073242,
|
|
"metrics/advantage_var": 697.8326416015625,
|
|
"regularization/advantage_var": 697.8326416015625,
|
|
"regularization/kl": 1.1618913412094116,
|
|
"rewards/chosen": 0.2475855268817547,
|
|
"rewards/margins": 0.8847968132063658,
|
|
"rewards/rejected": -0.6372112863246111,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 14.01774787902832,
|
|
"kl": 12.346087455749512,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36277728.84867395,
|
|
"logits/rejected": -36667039.44913928,
|
|
"logps/chosen": -460.0459243369735,
|
|
"logps/rejected": -373.7971439749609,
|
|
"loss": 0.5197,
|
|
"loss/base_kto": 3.16581392288208,
|
|
"loss/total_with_kl": 4.157769680023193,
|
|
"metrics/advantage_var": 595.7692260742188,
|
|
"regularization/advantage_var": 595.7692260742188,
|
|
"regularization/kl": 0.9919557571411133,
|
|
"rewards/chosen": 0.4603787569471529,
|
|
"rewards/margins": 0.8959217157634185,
|
|
"rewards/rejected": -0.43554295881626565,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 17.604755401611328,
|
|
"kl": 10.787038803100586,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35393958.10687023,
|
|
"logits/rejected": -36645922.4064,
|
|
"logps/chosen": -461.3892175572519,
|
|
"logps/rejected": -370.68335,
|
|
"loss": 0.5349,
|
|
"loss/base_kto": 3.1907060146331787,
|
|
"loss/total_with_kl": 4.279114723205566,
|
|
"metrics/advantage_var": 653.69873046875,
|
|
"regularization/advantage_var": 653.69873046875,
|
|
"regularization/kl": 1.088408350944519,
|
|
"rewards/chosen": 0.46098511674021947,
|
|
"rewards/margins": 0.8803063081464695,
|
|
"rewards/rejected": -0.41932119140625,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 28.774839401245117,
|
|
"kl": 15.302284240722656,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -37184880.145015106,
|
|
"logits/rejected": -37203610.097087376,
|
|
"logps/chosen": -506.91814577039275,
|
|
"logps/rejected": -382.5256877022654,
|
|
"loss": 0.5375,
|
|
"loss/base_kto": 3.16581130027771,
|
|
"loss/total_with_kl": 4.299928188323975,
|
|
"metrics/advantage_var": 681.1513061523438,
|
|
"regularization/advantage_var": 681.1513061523438,
|
|
"regularization/kl": 1.1341170072555542,
|
|
"rewards/chosen": 0.599076101066843,
|
|
"rewards/margins": 0.8863653470673613,
|
|
"rewards/rejected": -0.2872892460005183,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 12.60871696472168,
|
|
"kl": 12.3494873046875,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34259159.578947365,
|
|
"logits/rejected": -35080630.85714286,
|
|
"logps/chosen": -488.53196957236844,
|
|
"logps/rejected": -376.5957961309524,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.1485283374786377,
|
|
"loss/total_with_kl": 4.483153343200684,
|
|
"metrics/advantage_var": 801.5763549804688,
|
|
"regularization/advantage_var": 801.5763549804688,
|
|
"regularization/kl": 1.3346246480941772,
|
|
"rewards/chosen": 0.4776663529245477,
|
|
"rewards/margins": 0.952251386522948,
|
|
"rewards/rejected": -0.4745850335984003,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 17.281497955322266,
|
|
"kl": 10.597274780273438,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35714048.0,
|
|
"logits/rejected": -37265164.487804875,
|
|
"logps/chosen": -488.4378004807692,
|
|
"logps/rejected": -372.6516053734756,
|
|
"loss": 0.5289,
|
|
"loss/base_kto": 3.154386281967163,
|
|
"loss/total_with_kl": 4.231297969818115,
|
|
"metrics/advantage_var": 646.7941284179688,
|
|
"regularization/advantage_var": 646.7941284179688,
|
|
"regularization/kl": 1.0769122838974,
|
|
"rewards/chosen": 0.5121666345840845,
|
|
"rewards/margins": 0.8904215715466774,
|
|
"rewards/rejected": -0.3782549369625929,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 18.517337799072266,
|
|
"kl": 11.049582481384277,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -36722954.71627907,
|
|
"logits/rejected": -38406360.088188976,
|
|
"logps/chosen": -491.9407945736434,
|
|
"logps/rejected": -379.6351131889764,
|
|
"loss": 0.5082,
|
|
"loss/base_kto": 3.207235336303711,
|
|
"loss/total_with_kl": 4.065227031707764,
|
|
"metrics/advantage_var": 515.3104248046875,
|
|
"regularization/advantage_var": 515.3104248046875,
|
|
"regularization/kl": 0.8579918146133423,
|
|
"rewards/chosen": 0.46582092758296995,
|
|
"rewards/margins": 0.8286288907965527,
|
|
"rewards/rejected": -0.36280796321358266,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 12.599120140075684,
|
|
"kl": 15.788786888122559,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35283677.70716511,
|
|
"logits/rejected": -36716287.19749216,
|
|
"logps/chosen": -474.21928543613706,
|
|
"logps/rejected": -393.32949647335425,
|
|
"loss": 0.5195,
|
|
"loss/base_kto": 3.2040252685546875,
|
|
"loss/total_with_kl": 4.156173229217529,
|
|
"metrics/advantage_var": 571.8604736328125,
|
|
"regularization/advantage_var": 571.8604736328125,
|
|
"regularization/kl": 0.9521476030349731,
|
|
"rewards/chosen": 0.5738645179249416,
|
|
"rewards/margins": 0.8790741172434908,
|
|
"rewards/rejected": -0.3052095993185492,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 14.024765014648438,
|
|
"kl": 18.037206649780273,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34794381.14375988,
|
|
"logits/rejected": -34839822.63987635,
|
|
"logps/chosen": -468.80677330173773,
|
|
"logps/rejected": -405.15938948995364,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.163182497024536,
|
|
"loss/total_with_kl": 4.438502788543701,
|
|
"metrics/advantage_var": 765.958251953125,
|
|
"regularization/advantage_var": 765.958251953125,
|
|
"regularization/kl": 1.2753204107284546,
|
|
"rewards/chosen": 0.5771619365866903,
|
|
"rewards/margins": 0.9175937749391827,
|
|
"rewards/rejected": -0.3404318383524923,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 17.971912384033203,
|
|
"kl": 13.64586353302002,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35411977.19760479,
|
|
"logits/rejected": -35166495.79084967,
|
|
"logps/chosen": -476.7110778443114,
|
|
"logps/rejected": -385.3743617238562,
|
|
"loss": 0.5063,
|
|
"loss/base_kto": 3.1725013256073,
|
|
"loss/total_with_kl": 4.050698280334473,
|
|
"metrics/advantage_var": 527.4454956054688,
|
|
"regularization/advantage_var": 527.4454956054688,
|
|
"regularization/kl": 0.8781967163085938,
|
|
"rewards/chosen": 0.5798879383566851,
|
|
"rewards/margins": 0.8787531631575429,
|
|
"rewards/rejected": -0.29886522480085786,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 13.165546417236328,
|
|
"kl": 20.489919662475586,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34604458.538345866,
|
|
"logits/rejected": -36270602.822764225,
|
|
"logps/chosen": -509.2143327067669,
|
|
"logps/rejected": -364.005081300813,
|
|
"loss": 0.5053,
|
|
"loss/base_kto": 3.206397771835327,
|
|
"loss/total_with_kl": 4.042151927947998,
|
|
"metrics/advantage_var": 501.9543151855469,
|
|
"regularization/advantage_var": 501.9543151855469,
|
|
"regularization/kl": 0.8357540369033813,
|
|
"rewards/chosen": 0.6431686516094924,
|
|
"rewards/margins": 0.8166307889713446,
|
|
"rewards/rejected": -0.17346213736185215,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 12.91707706451416,
|
|
"kl": 21.993675231933594,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35183648.61146497,
|
|
"logits/rejected": -35695361.57055215,
|
|
"logps/chosen": -490.61962579617835,
|
|
"logps/rejected": -376.44924271472394,
|
|
"loss": 0.5275,
|
|
"loss/base_kto": 3.1577212810516357,
|
|
"loss/total_with_kl": 4.2201361656188965,
|
|
"metrics/advantage_var": 638.0870361328125,
|
|
"regularization/advantage_var": 638.0870361328125,
|
|
"regularization/kl": 1.0624150037765503,
|
|
"rewards/chosen": 0.7239522751729199,
|
|
"rewards/margins": 0.9027313857596363,
|
|
"rewards/rejected": -0.17877911058671636,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 15.724650382995605,
|
|
"kl": 16.417760848999023,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -36217919.49125596,
|
|
"logits/rejected": -37836051.2688172,
|
|
"logps/chosen": -503.1971383147854,
|
|
"logps/rejected": -366.30568356374806,
|
|
"loss": 0.4982,
|
|
"loss/base_kto": 3.1603543758392334,
|
|
"loss/total_with_kl": 3.985971450805664,
|
|
"metrics/advantage_var": 495.8663024902344,
|
|
"regularization/advantage_var": 495.8663024902344,
|
|
"regularization/kl": 0.8256174325942993,
|
|
"rewards/chosen": 0.5560551629726004,
|
|
"rewards/margins": 0.8365336448454402,
|
|
"rewards/rejected": -0.28047848187283986,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 8.864245414733887,
|
|
"kl": 18.243162155151367,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34416254.00623053,
|
|
"logits/rejected": -35867651.210031345,
|
|
"logps/chosen": -467.94256230529595,
|
|
"logps/rejected": -376.60295846394985,
|
|
"loss": 0.5065,
|
|
"loss/base_kto": 3.1706998348236084,
|
|
"loss/total_with_kl": 4.052399158477783,
|
|
"metrics/advantage_var": 529.5491943359375,
|
|
"regularization/advantage_var": 529.5491943359375,
|
|
"regularization/kl": 0.8816993832588196,
|
|
"rewards/chosen": 0.6203484936295268,
|
|
"rewards/margins": 0.8738607416059914,
|
|
"rewards/rejected": -0.25351224797646454,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 18.818096160888672,
|
|
"kl": 15.484634399414062,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34946939.65751211,
|
|
"logits/rejected": -35976376.35098336,
|
|
"logps/chosen": -469.2755452342488,
|
|
"logps/rejected": -377.5408472012103,
|
|
"loss": 0.4996,
|
|
"loss/base_kto": 3.171642780303955,
|
|
"loss/total_with_kl": 3.996892213821411,
|
|
"metrics/advantage_var": 495.6452331542969,
|
|
"regularization/advantage_var": 495.6452331542969,
|
|
"regularization/kl": 0.8252493143081665,
|
|
"rewards/chosen": 0.5336291878597031,
|
|
"rewards/margins": 0.8582243059336441,
|
|
"rewards/rejected": -0.324595118073941,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 37.44119644165039,
|
|
"kl": 11.217509269714355,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34135380.24801272,
|
|
"logits/rejected": -35630465.376344085,
|
|
"logps/chosen": -484.7997813990461,
|
|
"logps/rejected": -376.2759216589862,
|
|
"loss": 0.5282,
|
|
"loss/base_kto": 3.1395866870880127,
|
|
"loss/total_with_kl": 4.225513458251953,
|
|
"metrics/advantage_var": 652.2083129882812,
|
|
"regularization/advantage_var": 652.2083129882812,
|
|
"regularization/kl": 1.08592689037323,
|
|
"rewards/chosen": 0.5123951825504273,
|
|
"rewards/margins": 0.926046468732743,
|
|
"rewards/rejected": -0.41365128618231567,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 39.47532653808594,
|
|
"kl": 12.592890739440918,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34648123.55411955,
|
|
"logits/rejected": -36117406.40242057,
|
|
"logps/chosen": -482.23046243941843,
|
|
"logps/rejected": -373.59852496217854,
|
|
"loss": 0.5135,
|
|
"loss/base_kto": 3.1341800689697266,
|
|
"loss/total_with_kl": 4.107840061187744,
|
|
"metrics/advantage_var": 584.7808227539062,
|
|
"regularization/advantage_var": 584.7808227539062,
|
|
"regularization/kl": 0.9736601114273071,
|
|
"rewards/chosen": 0.5263789212376313,
|
|
"rewards/margins": 0.9207766510622342,
|
|
"rewards/rejected": -0.39439772982460286,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 12.804630279541016,
|
|
"kl": 9.709197044372559,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34007461.453376204,
|
|
"logits/rejected": -35523036.20668693,
|
|
"logps/chosen": -498.3818830385852,
|
|
"logps/rejected": -345.47091090425533,
|
|
"loss": 0.4968,
|
|
"loss/base_kto": 3.1976964473724365,
|
|
"loss/total_with_kl": 3.974379777908325,
|
|
"metrics/advantage_var": 466.4764709472656,
|
|
"regularization/advantage_var": 466.4764709472656,
|
|
"regularization/kl": 0.7766832709312439,
|
|
"rewards/chosen": 0.4751230712105607,
|
|
"rewards/margins": 0.8474625227772163,
|
|
"rewards/rejected": -0.3723394515666556,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 22.580102920532227,
|
|
"kl": 13.041857719421387,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34118773.1589896,
|
|
"logits/rejected": -37289167.49917628,
|
|
"logps/chosen": -502.77693164933135,
|
|
"logps/rejected": -363.8554880560132,
|
|
"loss": 0.5379,
|
|
"loss/base_kto": 3.1549394130706787,
|
|
"loss/total_with_kl": 4.303187847137451,
|
|
"metrics/advantage_var": 689.6389770507812,
|
|
"regularization/advantage_var": 689.6389770507812,
|
|
"regularization/kl": 1.1482489109039307,
|
|
"rewards/chosen": 0.6239117059806835,
|
|
"rewards/margins": 0.9157528138723381,
|
|
"rewards/rejected": -0.2918411078916546,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 294.7341003417969,
|
|
"kl": 13.90600872039795,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34551348.63551402,
|
|
"logits/rejected": -36423211.335423194,
|
|
"logps/chosen": -462.7169003115265,
|
|
"logps/rejected": -376.18399784482756,
|
|
"loss": 0.6041,
|
|
"loss/base_kto": 3.23050856590271,
|
|
"loss/total_with_kl": 4.832540035247803,
|
|
"metrics/advantage_var": 962.1810913085938,
|
|
"regularization/advantage_var": 962.1810913085938,
|
|
"regularization/kl": 1.6020317077636719,
|
|
"rewards/chosen": 0.561777560510368,
|
|
"rewards/margins": 0.869574050740151,
|
|
"rewards/rejected": -0.307796490229783,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 20.57028579711914,
|
|
"kl": 13.543728828430176,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33880727.16190476,
|
|
"logits/rejected": -36298474.732307695,
|
|
"logps/chosen": -507.5727182539683,
|
|
"logps/rejected": -376.46466346153846,
|
|
"loss": 0.5345,
|
|
"loss/base_kto": 3.1752588748931885,
|
|
"loss/total_with_kl": 4.276364326477051,
|
|
"metrics/advantage_var": 661.3244018554688,
|
|
"regularization/advantage_var": 661.3244018554688,
|
|
"regularization/kl": 1.1011050939559937,
|
|
"rewards/chosen": 0.578230697389633,
|
|
"rewards/margins": 0.9060270700233468,
|
|
"rewards/rejected": -0.3277963726337139,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 22.362438201904297,
|
|
"kl": 13.31640625,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -33722781.50715421,
|
|
"logits/rejected": -35482087.61904762,
|
|
"logps/chosen": -482.0680147058824,
|
|
"logps/rejected": -355.771001344086,
|
|
"loss": 0.498,
|
|
"loss/base_kto": 3.190498113632202,
|
|
"loss/total_with_kl": 3.9841275215148926,
|
|
"metrics/advantage_var": 476.6542663574219,
|
|
"regularization/advantage_var": 476.6542663574219,
|
|
"regularization/kl": 0.7936292886734009,
|
|
"rewards/chosen": 0.532119823752981,
|
|
"rewards/margins": 0.8617982669233438,
|
|
"rewards/rejected": -0.3296784431703629,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 10.12584114074707,
|
|
"kl": 9.132817268371582,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34739407.372056514,
|
|
"logits/rejected": -35731210.7496112,
|
|
"logps/chosen": -473.11548273155415,
|
|
"logps/rejected": -369.72633164852255,
|
|
"loss": 0.513,
|
|
"loss/base_kto": 3.213224411010742,
|
|
"loss/total_with_kl": 4.104243755340576,
|
|
"metrics/advantage_var": 535.1466674804688,
|
|
"regularization/advantage_var": 535.1466674804688,
|
|
"regularization/kl": 0.8910191655158997,
|
|
"rewards/chosen": 0.41781879706510006,
|
|
"rewards/margins": 0.8382309466872326,
|
|
"rewards/rejected": -0.4204121496221326,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 11.648301124572754,
|
|
"kl": 13.205253601074219,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33042906.80680062,
|
|
"logits/rejected": -34789291.87993681,
|
|
"logps/chosen": -469.2457496136012,
|
|
"logps/rejected": -368.17145537124804,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.186049222946167,
|
|
"loss/total_with_kl": 4.12746524810791,
|
|
"metrics/advantage_var": 565.4149780273438,
|
|
"regularization/advantage_var": 565.4149780273438,
|
|
"regularization/kl": 0.9414157867431641,
|
|
"rewards/chosen": 0.5633167579332496,
|
|
"rewards/margins": 0.8688969814626977,
|
|
"rewards/rejected": -0.3055802235294481,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 9.580194473266602,
|
|
"kl": 14.520026206970215,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34545826.90909091,
|
|
"logits/rejected": -35603428.240963854,
|
|
"logps/chosen": -483.2869318181818,
|
|
"logps/rejected": -386.94423004518075,
|
|
"loss": 0.5089,
|
|
"loss/base_kto": 3.1537270545959473,
|
|
"loss/total_with_kl": 4.070878505706787,
|
|
"metrics/advantage_var": 550.841796875,
|
|
"regularization/advantage_var": 550.841796875,
|
|
"regularization/kl": 0.9171515703201294,
|
|
"rewards/chosen": 0.5488103395932681,
|
|
"rewards/margins": 0.909023332066089,
|
|
"rewards/rejected": -0.36021299247282096,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 34.79518127441406,
|
|
"kl": 6.8231730461120605,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34463994.90352221,
|
|
"logits/rejected": -35986306.24561404,
|
|
"logps/chosen": -483.4157733537519,
|
|
"logps/rejected": -375.83243620414675,
|
|
"loss": 0.521,
|
|
"loss/base_kto": 3.204749822616577,
|
|
"loss/total_with_kl": 4.167901515960693,
|
|
"metrics/advantage_var": 578.4695434570312,
|
|
"regularization/advantage_var": 578.4695434570312,
|
|
"regularization/kl": 0.963151752948761,
|
|
"rewards/chosen": 0.3909355969743013,
|
|
"rewards/margins": 0.9083980277427173,
|
|
"rewards/rejected": -0.517462430768416,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 13.085524559020996,
|
|
"kl": 15.920722961425781,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34013177.71779141,
|
|
"logits/rejected": -35884775.541401275,
|
|
"logps/chosen": -482.31643021472394,
|
|
"logps/rejected": -382.711236066879,
|
|
"loss": 0.5202,
|
|
"loss/base_kto": 3.112375259399414,
|
|
"loss/total_with_kl": 4.161748886108398,
|
|
"metrics/advantage_var": 630.25439453125,
|
|
"regularization/advantage_var": 630.25439453125,
|
|
"regularization/kl": 1.0493735074996948,
|
|
"rewards/chosen": 0.6457256480960026,
|
|
"rewards/margins": 0.9098842979050442,
|
|
"rewards/rejected": -0.2641586498090416,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 14.14444351196289,
|
|
"kl": 14.376886367797852,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34573030.68131868,
|
|
"logits/rejected": -35671637.20062208,
|
|
"logps/chosen": -473.5124116954474,
|
|
"logps/rejected": -392.93164366251943,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.2205846309661865,
|
|
"loss/total_with_kl": 4.144931316375732,
|
|
"metrics/advantage_var": 555.1632690429688,
|
|
"regularization/advantage_var": 555.1632690429688,
|
|
"regularization/kl": 0.9243467450141907,
|
|
"rewards/chosen": 0.5032569969093407,
|
|
"rewards/margins": 0.8434753584865422,
|
|
"rewards/rejected": -0.3402183615772016,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 12.696953773498535,
|
|
"kl": 16.796072006225586,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34975784.15686274,
|
|
"logits/rejected": -35659706.29497569,
|
|
"logps/chosen": -472.6144419306184,
|
|
"logps/rejected": -349.7544064019449,
|
|
"loss": 0.5166,
|
|
"loss/base_kto": 3.1975579261779785,
|
|
"loss/total_with_kl": 4.13291072845459,
|
|
"metrics/advantage_var": 561.7733764648438,
|
|
"regularization/advantage_var": 561.7733764648438,
|
|
"regularization/kl": 0.9353527426719666,
|
|
"rewards/chosen": 0.5741287162401018,
|
|
"rewards/margins": 0.8371576199972559,
|
|
"rewards/rejected": -0.2630289037571541,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 7.896336555480957,
|
|
"kl": 15.931853294372559,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34911612.52950558,
|
|
"logits/rejected": -35763725.32924962,
|
|
"logps/chosen": -483.26226076555025,
|
|
"logps/rejected": -358.62016653905056,
|
|
"loss": 0.5232,
|
|
"loss/base_kto": 3.2096874713897705,
|
|
"loss/total_with_kl": 4.185441493988037,
|
|
"metrics/advantage_var": 586.0383911132812,
|
|
"regularization/advantage_var": 586.0383911132812,
|
|
"regularization/kl": 0.9757537841796875,
|
|
"rewards/chosen": 0.5678250496847588,
|
|
"rewards/margins": 0.84200268273953,
|
|
"rewards/rejected": -0.2741776330547713,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 16.918319702148438,
|
|
"kl": 18.675029754638672,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -33134305.720430106,
|
|
"logits/rejected": -34378620.13354531,
|
|
"logps/chosen": -474.04877112135176,
|
|
"logps/rejected": -351.6985790937997,
|
|
"loss": 0.4963,
|
|
"loss/base_kto": 3.1137306690216064,
|
|
"loss/total_with_kl": 3.9700634479522705,
|
|
"metrics/advantage_var": 514.3140258789062,
|
|
"regularization/advantage_var": 514.3140258789062,
|
|
"regularization/kl": 0.8563327789306641,
|
|
"rewards/chosen": 0.7106950469830069,
|
|
"rewards/margins": 0.9445379300703106,
|
|
"rewards/rejected": -0.23384288308730375,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 19.034807205200195,
|
|
"kl": 13.459898948669434,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34716878.06811146,
|
|
"logits/rejected": -35929924.643533126,
|
|
"logps/chosen": -469.08126934984523,
|
|
"logps/rejected": -394.77743000788644,
|
|
"loss": 0.5015,
|
|
"loss/base_kto": 3.2017509937286377,
|
|
"loss/total_with_kl": 4.01181173324585,
|
|
"metrics/advantage_var": 486.5226745605469,
|
|
"regularization/advantage_var": 486.5226745605469,
|
|
"regularization/kl": 0.8100603222846985,
|
|
"rewards/chosen": 0.4974288231817192,
|
|
"rewards/margins": 0.8555690646421885,
|
|
"rewards/rejected": -0.35814024146046924,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 27.855440139770508,
|
|
"kl": 18.5737247467041,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34781177.69846154,
|
|
"logits/rejected": -35781245.155555554,
|
|
"logps/chosen": -493.2761538461538,
|
|
"logps/rejected": -390.83214285714286,
|
|
"loss": 0.5013,
|
|
"loss/base_kto": 3.1283771991729736,
|
|
"loss/total_with_kl": 4.010641574859619,
|
|
"metrics/advantage_var": 529.8883666992188,
|
|
"regularization/advantage_var": 529.8883666992188,
|
|
"regularization/kl": 0.8822641372680664,
|
|
"rewards/chosen": 0.6580948110727164,
|
|
"rewards/margins": 0.9275835302257422,
|
|
"rewards/rejected": -0.2694887191530258,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 23.49469757080078,
|
|
"kl": 16.161033630371094,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35162405.50318471,
|
|
"logits/rejected": -37646178.944785275,
|
|
"logps/chosen": -474.16142515923565,
|
|
"logps/rejected": -371.1085601993865,
|
|
"loss": 0.4972,
|
|
"loss/base_kto": 3.156907558441162,
|
|
"loss/total_with_kl": 3.977708578109741,
|
|
"metrics/advantage_var": 492.9736022949219,
|
|
"regularization/advantage_var": 492.9736022949219,
|
|
"regularization/kl": 0.8208009600639343,
|
|
"rewards/chosen": 0.6076517287333301,
|
|
"rewards/margins": 0.8742902212473495,
|
|
"rewards/rejected": -0.2666384925140194,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 22.5130558013916,
|
|
"kl": 16.55893898010254,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33921503.255813956,
|
|
"logits/rejected": -35077348.324324325,
|
|
"logps/chosen": -476.2427779796512,
|
|
"logps/rejected": -374.5779930320946,
|
|
"loss": 0.5097,
|
|
"loss/base_kto": 3.1985280513763428,
|
|
"loss/total_with_kl": 4.077399730682373,
|
|
"metrics/advantage_var": 527.8506469726562,
|
|
"regularization/advantage_var": 527.8506469726562,
|
|
"regularization/kl": 0.8788713812828064,
|
|
"rewards/chosen": 0.6884309192036473,
|
|
"rewards/margins": 0.8503661326534863,
|
|
"rewards/rejected": -0.161935213449839,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 21.97184944152832,
|
|
"kl": 19.698619842529297,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35926641.144674085,
|
|
"logits/rejected": -36667335.37327189,
|
|
"logps/chosen": -484.2610790937997,
|
|
"logps/rejected": -388.83419738863284,
|
|
"loss": 0.5118,
|
|
"loss/base_kto": 3.1325860023498535,
|
|
"loss/total_with_kl": 4.094533920288086,
|
|
"metrics/advantage_var": 577.74658203125,
|
|
"regularization/advantage_var": 577.74658203125,
|
|
"regularization/kl": 0.961948037147522,
|
|
"rewards/chosen": 0.7117422131172992,
|
|
"rewards/margins": 0.9445239236467731,
|
|
"rewards/rejected": -0.23278171052947388,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 12.882615089416504,
|
|
"kl": 15.500436782836914,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -33707717.4379085,
|
|
"logits/rejected": -34998179.74774775,
|
|
"logps/chosen": -487.6294934640523,
|
|
"logps/rejected": -376.18308933933935,
|
|
"loss": 0.47,
|
|
"loss/base_kto": 3.1957995891571045,
|
|
"loss/total_with_kl": 3.759805679321289,
|
|
"metrics/advantage_var": 338.74224853515625,
|
|
"regularization/advantage_var": 338.74224853515625,
|
|
"regularization/kl": 0.5640057921409607,
|
|
"rewards/chosen": 0.512441248675577,
|
|
"rewards/margins": 0.8036668080362073,
|
|
"rewards/rejected": -0.29122555936063016,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 10.57919692993164,
|
|
"kl": 13.648648262023926,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34193498.75332349,
|
|
"logits/rejected": -34592068.35157546,
|
|
"logps/chosen": -457.49312223042836,
|
|
"logps/rejected": -381.55161691542287,
|
|
"loss": 0.4713,
|
|
"loss/base_kto": 3.136463165283203,
|
|
"loss/total_with_kl": 3.770439624786377,
|
|
"metrics/advantage_var": 380.7665710449219,
|
|
"regularization/advantage_var": 380.7665710449219,
|
|
"regularization/kl": 0.6339762806892395,
|
|
"rewards/chosen": 0.6271113025208872,
|
|
"rewards/margins": 0.9073408615130358,
|
|
"rewards/rejected": -0.28022955899214863,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 9.71323013305664,
|
|
"kl": 19.46393394470215,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33426026.264150944,
|
|
"logits/rejected": -36114492.42236025,
|
|
"logps/chosen": -497.7997248427673,
|
|
"logps/rejected": -360.94589479813664,
|
|
"loss": 0.4692,
|
|
"loss/base_kto": 3.1342921257019043,
|
|
"loss/total_with_kl": 3.753847122192383,
|
|
"metrics/advantage_var": 372.105224609375,
|
|
"regularization/advantage_var": 372.105224609375,
|
|
"regularization/kl": 0.6195551753044128,
|
|
"rewards/chosen": 0.6923610279395146,
|
|
"rewards/margins": 0.8642818043067021,
|
|
"rewards/rejected": -0.1719207763671875,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 13.907105445861816,
|
|
"kl": 26.03472900390625,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34063458.79326187,
|
|
"logits/rejected": -36368384.0,
|
|
"logps/chosen": -481.4810490045942,
|
|
"logps/rejected": -384.0566437400319,
|
|
"loss": 0.4609,
|
|
"loss/base_kto": 3.116948127746582,
|
|
"loss/total_with_kl": 3.6871414184570312,
|
|
"metrics/advantage_var": 342.4584655761719,
|
|
"regularization/advantage_var": 342.4584655761719,
|
|
"regularization/kl": 0.5701932907104492,
|
|
"rewards/chosen": 0.7419502760695827,
|
|
"rewards/margins": 0.8515068726947572,
|
|
"rewards/rejected": -0.10955659662517445,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 11.830923080444336,
|
|
"kl": 15.881300926208496,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34649600.80376766,
|
|
"logits/rejected": -34635619.93157076,
|
|
"logps/chosen": -481.37102629513345,
|
|
"logps/rejected": -373.75758164852255,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.110450506210327,
|
|
"loss/total_with_kl": 3.693946123123169,
|
|
"metrics/advantage_var": 350.4477233886719,
|
|
"regularization/advantage_var": 350.4477233886719,
|
|
"regularization/kl": 0.5834954380989075,
|
|
"rewards/chosen": 0.648567618904533,
|
|
"rewards/margins": 0.891513929229669,
|
|
"rewards/rejected": -0.24294631032513608,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 12.304580688476562,
|
|
"kl": 13.850641250610352,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34127951.79220779,
|
|
"logits/rejected": -34919266.69879518,
|
|
"logps/chosen": -471.86444805194805,
|
|
"logps/rejected": -370.60791603915663,
|
|
"loss": 0.4674,
|
|
"loss/base_kto": 3.119288206100464,
|
|
"loss/total_with_kl": 3.739403486251831,
|
|
"metrics/advantage_var": 372.4416809082031,
|
|
"regularization/advantage_var": 372.4416809082031,
|
|
"regularization/kl": 0.6201154589653015,
|
|
"rewards/chosen": 0.5691945459935572,
|
|
"rewards/margins": 0.8991698350281134,
|
|
"rewards/rejected": -0.3299752890345562,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 15.00333309173584,
|
|
"kl": 10.780613899230957,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34084100.770186335,
|
|
"logits/rejected": -36909487.49685535,
|
|
"logps/chosen": -489.0129076086956,
|
|
"logps/rejected": -361.6150501179245,
|
|
"loss": 0.4788,
|
|
"loss/base_kto": 3.13610577583313,
|
|
"loss/total_with_kl": 3.8302910327911377,
|
|
"metrics/advantage_var": 416.9278869628906,
|
|
"regularization/advantage_var": 416.9278869628906,
|
|
"regularization/kl": 0.6941849589347839,
|
|
"rewards/chosen": 0.5348799806203901,
|
|
"rewards/margins": 0.894672246122552,
|
|
"rewards/rejected": -0.3597922655021619,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 10.705175399780273,
|
|
"kl": 12.834040641784668,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34572940.65934066,
|
|
"logits/rejected": -36874281.4059098,
|
|
"logps/chosen": -524.3274627158555,
|
|
"logps/rejected": -371.9184000777605,
|
|
"loss": 0.4585,
|
|
"loss/base_kto": 3.0802323818206787,
|
|
"loss/total_with_kl": 3.6678178310394287,
|
|
"metrics/advantage_var": 352.9039001464844,
|
|
"regularization/advantage_var": 352.9039001464844,
|
|
"regularization/kl": 0.5875850915908813,
|
|
"rewards/chosen": 0.5883653961133732,
|
|
"rewards/margins": 0.9297093722187266,
|
|
"rewards/rejected": -0.34134397610535333,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 7.3886027336120605,
|
|
"kl": 15.603533744812012,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34152773.178625956,
|
|
"logits/rejected": -36127411.4048,
|
|
"logps/chosen": -473.0314885496183,
|
|
"logps/rejected": -391.2823,
|
|
"loss": 0.4593,
|
|
"loss/base_kto": 3.057488203048706,
|
|
"loss/total_with_kl": 3.67455792427063,
|
|
"metrics/advantage_var": 370.61236572265625,
|
|
"regularization/advantage_var": 370.61236572265625,
|
|
"regularization/kl": 0.6170695424079895,
|
|
"rewards/chosen": 0.6530079150017891,
|
|
"rewards/margins": 0.9697260058221016,
|
|
"rewards/rejected": -0.3167180908203125,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 18.078439712524414,
|
|
"kl": 13.160568237304688,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34044216.67175572,
|
|
"logits/rejected": -35952535.1424,
|
|
"logps/chosen": -492.331965648855,
|
|
"logps/rejected": -364.4094,
|
|
"loss": 0.4659,
|
|
"loss/base_kto": 3.1652474403381348,
|
|
"loss/total_with_kl": 3.7272117137908936,
|
|
"metrics/advantage_var": 337.51605224609375,
|
|
"regularization/advantage_var": 337.51605224609375,
|
|
"regularization/kl": 0.561964213848114,
|
|
"rewards/chosen": 0.5588921233897901,
|
|
"rewards/margins": 0.84985140073354,
|
|
"rewards/rejected": -0.29095927734375,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 7.16786527633667,
|
|
"kl": 19.17708396911621,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34342403.92036753,
|
|
"logits/rejected": -34944413.19298246,
|
|
"logps/chosen": -442.2739758805513,
|
|
"logps/rejected": -371.99456738437004,
|
|
"loss": 0.4596,
|
|
"loss/base_kto": 3.1005120277404785,
|
|
"loss/total_with_kl": 3.6771087646484375,
|
|
"metrics/advantage_var": 346.304443359375,
|
|
"regularization/advantage_var": 346.304443359375,
|
|
"regularization/kl": 0.5765969157218933,
|
|
"rewards/chosen": 0.6813067075486935,
|
|
"rewards/margins": 0.9012701098176827,
|
|
"rewards/rejected": -0.21996340226898922,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 11.32540512084961,
|
|
"kl": 21.220792770385742,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -35406720.3987539,
|
|
"logits/rejected": -36062256.15047022,
|
|
"logps/chosen": -486.7122760903427,
|
|
"logps/rejected": -391.71177018025077,
|
|
"loss": 0.4586,
|
|
"loss/base_kto": 3.080542802810669,
|
|
"loss/total_with_kl": 3.6685128211975098,
|
|
"metrics/advantage_var": 353.13525390625,
|
|
"regularization/advantage_var": 353.13525390625,
|
|
"regularization/kl": 0.5879701972007751,
|
|
"rewards/chosen": 0.6702795296071846,
|
|
"rewards/margins": 0.8975216914685924,
|
|
"rewards/rejected": -0.22724216186140772,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 13.638971328735352,
|
|
"kl": 17.907649993896484,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -36546572.740279935,
|
|
"logits/rejected": -35178878.593406595,
|
|
"logps/chosen": -487.7169031881804,
|
|
"logps/rejected": -367.28115188383043,
|
|
"loss": 0.4589,
|
|
"loss/base_kto": 3.0799336433410645,
|
|
"loss/total_with_kl": 3.671400785446167,
|
|
"metrics/advantage_var": 355.23553466796875,
|
|
"regularization/advantage_var": 355.23553466796875,
|
|
"regularization/kl": 0.5914670825004578,
|
|
"rewards/chosen": 0.6589480766426905,
|
|
"rewards/margins": 0.9246811816898476,
|
|
"rewards/rejected": -0.2657331050471571,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 14.756328582763672,
|
|
"kl": 18.99579429626465,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34947643.23296355,
|
|
"logits/rejected": -36761795.64869029,
|
|
"logps/chosen": -463.9481973058637,
|
|
"logps/rejected": -354.99727946841296,
|
|
"loss": 0.464,
|
|
"loss/base_kto": 3.133209466934204,
|
|
"loss/total_with_kl": 3.71171498298645,
|
|
"metrics/advantage_var": 347.45062255859375,
|
|
"regularization/advantage_var": 347.45062255859375,
|
|
"regularization/kl": 0.5785053372383118,
|
|
"rewards/chosen": 0.5697169477701565,
|
|
"rewards/margins": 0.8377728209885694,
|
|
"rewards/rejected": -0.26805587321841295,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 14.190794944763184,
|
|
"kl": 17.259674072265625,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35423563.627172194,
|
|
"logits/rejected": -36394916.20401855,
|
|
"logps/chosen": -454.6295912322275,
|
|
"logps/rejected": -395.64364374034005,
|
|
"loss": 0.4631,
|
|
"loss/base_kto": 3.128011465072632,
|
|
"loss/total_with_kl": 3.704951524734497,
|
|
"metrics/advantage_var": 346.5107727050781,
|
|
"regularization/advantage_var": 346.5107727050781,
|
|
"regularization/kl": 0.5769404768943787,
|
|
"rewards/chosen": 0.5691574602895438,
|
|
"rewards/margins": 0.8692318062039807,
|
|
"rewards/rejected": -0.3000743459144368,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 11.9459810256958,
|
|
"kl": 12.720152854919434,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35945476.777604975,
|
|
"logits/rejected": -35812575.447409734,
|
|
"logps/chosen": -470.9396870139969,
|
|
"logps/rejected": -392.8932986656201,
|
|
"loss": 0.4593,
|
|
"loss/base_kto": 3.111800193786621,
|
|
"loss/total_with_kl": 3.6745898723602295,
|
|
"metrics/advantage_var": 338.0118103027344,
|
|
"regularization/advantage_var": 338.0118103027344,
|
|
"regularization/kl": 0.5627896785736084,
|
|
"rewards/chosen": 0.5418180996670878,
|
|
"rewards/margins": 0.8896113673647918,
|
|
"rewards/rejected": -0.3477932676977041,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 9.321317672729492,
|
|
"kl": 17.556133270263672,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -32271426.990654204,
|
|
"logits/rejected": -35846343.02194358,
|
|
"logps/chosen": -465.73481308411215,
|
|
"logps/rejected": -385.6867163009404,
|
|
"loss": 0.4578,
|
|
"loss/base_kto": 3.1206257343292236,
|
|
"loss/total_with_kl": 3.6624162197113037,
|
|
"metrics/advantage_var": 325.39959716796875,
|
|
"regularization/advantage_var": 325.39959716796875,
|
|
"regularization/kl": 0.5417903065681458,
|
|
"rewards/chosen": 0.6162986874209014,
|
|
"rewards/margins": 0.874495168390178,
|
|
"rewards/rejected": -0.25819648096927655,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 11.190893173217773,
|
|
"kl": 17.787485122680664,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34321935.61476725,
|
|
"logits/rejected": -36091034.30136986,
|
|
"logps/chosen": -473.31335272873196,
|
|
"logps/rejected": -383.24210426179604,
|
|
"loss": 0.4655,
|
|
"loss/base_kto": 3.11572265625,
|
|
"loss/total_with_kl": 3.7238259315490723,
|
|
"metrics/advantage_var": 365.2272033691406,
|
|
"regularization/advantage_var": 365.2272033691406,
|
|
"regularization/kl": 0.6081032752990723,
|
|
"rewards/chosen": 0.6468030216223164,
|
|
"rewards/margins": 0.874947465546522,
|
|
"rewards/rejected": -0.22814444392420566,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 11.218920707702637,
|
|
"kl": 13.033807754516602,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35571392.640915595,
|
|
"logits/rejected": -36281053.19104991,
|
|
"logps/chosen": -480.8800965665236,
|
|
"logps/rejected": -388.6537489242685,
|
|
"loss": 0.4603,
|
|
"loss/base_kto": 3.087879180908203,
|
|
"loss/total_with_kl": 3.6822800636291504,
|
|
"metrics/advantage_var": 356.99755859375,
|
|
"regularization/advantage_var": 356.99755859375,
|
|
"regularization/kl": 0.5944010019302368,
|
|
"rewards/chosen": 0.6077811005118919,
|
|
"rewards/margins": 0.9137831149872039,
|
|
"rewards/rejected": -0.30600201447531195,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 10.710734367370605,
|
|
"kl": 19.303434371948242,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -35323335.48051948,
|
|
"logits/rejected": -37117396.81927711,
|
|
"logps/chosen": -529.1298701298701,
|
|
"logps/rejected": -361.406438253012,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.061537027359009,
|
|
"loss/total_with_kl": 3.693446397781372,
|
|
"metrics/advantage_var": 379.52490234375,
|
|
"regularization/advantage_var": 379.52490234375,
|
|
"regularization/kl": 0.6319090127944946,
|
|
"rewards/chosen": 0.6770719057553775,
|
|
"rewards/margins": 0.9370477757269631,
|
|
"rewards/rejected": -0.25997586997158556,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 9.4447603225708,
|
|
"kl": 17.10456657409668,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33333655.01577287,
|
|
"logits/rejected": -35455067.9380805,
|
|
"logps/chosen": -471.5881309148265,
|
|
"logps/rejected": -356.6826141640867,
|
|
"loss": 0.4577,
|
|
"loss/base_kto": 3.0967190265655518,
|
|
"loss/total_with_kl": 3.6612699031829834,
|
|
"metrics/advantage_var": 339.0694885253906,
|
|
"regularization/advantage_var": 339.0694885253906,
|
|
"regularization/kl": 0.5645507574081421,
|
|
"rewards/chosen": 0.659497185836455,
|
|
"rewards/margins": 0.9093857447113824,
|
|
"rewards/rejected": -0.24988855887492745,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 14.468193054199219,
|
|
"kl": 17.700286865234375,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35349790.848200314,
|
|
"logits/rejected": -34952244.7176287,
|
|
"logps/chosen": -459.523082942097,
|
|
"logps/rejected": -382.5817570202808,
|
|
"loss": 0.4593,
|
|
"loss/base_kto": 3.1270058155059814,
|
|
"loss/total_with_kl": 3.674060583114624,
|
|
"metrics/advantage_var": 328.5613098144531,
|
|
"regularization/advantage_var": 328.5613098144531,
|
|
"regularization/kl": 0.5470545887947083,
|
|
"rewards/chosen": 0.6088742538237236,
|
|
"rewards/margins": 0.8522961208253568,
|
|
"rewards/rejected": -0.24342186700163318,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 12.740854263305664,
|
|
"kl": 16.302776336669922,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -35310003.54751131,
|
|
"logits/rejected": -36473910.76823339,
|
|
"logps/chosen": -472.34455128205127,
|
|
"logps/rejected": -368.706619732577,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.1101086139678955,
|
|
"loss/total_with_kl": 3.693207263946533,
|
|
"metrics/advantage_var": 350.2093811035156,
|
|
"regularization/advantage_var": 350.2093811035156,
|
|
"regularization/kl": 0.5830985903739929,
|
|
"rewards/chosen": 0.6291640613952912,
|
|
"rewards/margins": 0.8746409804118279,
|
|
"rewards/rejected": -0.24547691901653668,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 16.703655242919922,
|
|
"kl": 18.12259864807129,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34409088.396284826,
|
|
"logits/rejected": -35824610.92744479,
|
|
"logps/chosen": -490.77699303405575,
|
|
"logps/rejected": -351.43954554416405,
|
|
"loss": 0.4692,
|
|
"loss/base_kto": 3.1362993717193604,
|
|
"loss/total_with_kl": 3.7539360523223877,
|
|
"metrics/advantage_var": 370.952880859375,
|
|
"regularization/advantage_var": 370.952880859375,
|
|
"regularization/kl": 0.6176366209983826,
|
|
"rewards/chosen": 0.6702606862543538,
|
|
"rewards/margins": 0.8669946122512484,
|
|
"rewards/rejected": -0.1967339259968947,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 11.437090873718262,
|
|
"kl": 17.222028732299805,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35721556.32047478,
|
|
"logits/rejected": -35439119.20792079,
|
|
"logps/chosen": -467.7715133531157,
|
|
"logps/rejected": -390.11785787953795,
|
|
"loss": 0.4737,
|
|
"loss/base_kto": 3.1542561054229736,
|
|
"loss/total_with_kl": 3.7895684242248535,
|
|
"metrics/advantage_var": 381.5690612792969,
|
|
"regularization/advantage_var": 381.5690612792969,
|
|
"regularization/kl": 0.6353124976158142,
|
|
"rewards/chosen": 0.6299233818620178,
|
|
"rewards/margins": 0.8236679909268061,
|
|
"rewards/rejected": -0.19374460906478833,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 17.85501480102539,
|
|
"kl": 23.396320343017578,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -35332604.69677419,
|
|
"logits/rejected": -36375266.52121212,
|
|
"logps/chosen": -479.29395161290324,
|
|
"logps/rejected": -393.30485321969695,
|
|
"loss": 0.4714,
|
|
"loss/base_kto": 3.076240301132202,
|
|
"loss/total_with_kl": 3.7709243297576904,
|
|
"metrics/advantage_var": 417.2275390625,
|
|
"regularization/advantage_var": 417.2275390625,
|
|
"regularization/kl": 0.694683849811554,
|
|
"rewards/chosen": 0.6907609508883569,
|
|
"rewards/margins": 0.920711410686534,
|
|
"rewards/rejected": -0.22995045979817708,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 11.629434585571289,
|
|
"kl": 20.134891510009766,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35256735.306501545,
|
|
"logits/rejected": -36730201.64037855,
|
|
"logps/chosen": -467.421536377709,
|
|
"logps/rejected": -382.1418079652997,
|
|
"loss": 0.4755,
|
|
"loss/base_kto": 3.1732566356658936,
|
|
"loss/total_with_kl": 3.8041954040527344,
|
|
"metrics/advantage_var": 378.9418640136719,
|
|
"regularization/advantage_var": 378.9418640136719,
|
|
"regularization/kl": 0.6309382319450378,
|
|
"rewards/chosen": 0.6376219002455011,
|
|
"rewards/margins": 0.8101594301276975,
|
|
"rewards/rejected": -0.17253752988219637,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 10.874473571777344,
|
|
"kl": 19.402769088745117,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33273703.354037266,
|
|
"logits/rejected": -35857182.591194965,
|
|
"logps/chosen": -501.71797360248445,
|
|
"logps/rejected": -352.53387873427675,
|
|
"loss": 0.4593,
|
|
"loss/base_kto": 3.080603837966919,
|
|
"loss/total_with_kl": 3.6747825145721436,
|
|
"metrics/advantage_var": 356.86395263671875,
|
|
"regularization/advantage_var": 356.86395263671875,
|
|
"regularization/kl": 0.5941784381866455,
|
|
"rewards/chosen": 0.723855249629998,
|
|
"rewards/margins": 0.8998491630046049,
|
|
"rewards/rejected": -0.17599391337460693,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 7.630674362182617,
|
|
"kl": 21.488651275634766,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34430249.71246006,
|
|
"logits/rejected": -37031719.92660551,
|
|
"logps/chosen": -489.7289337060703,
|
|
"logps/rejected": -377.8819762996942,
|
|
"loss": 0.4673,
|
|
"loss/base_kto": 3.0745832920074463,
|
|
"loss/total_with_kl": 3.7381234169006348,
|
|
"metrics/advantage_var": 398.52264404296875,
|
|
"regularization/advantage_var": 398.52264404296875,
|
|
"regularization/kl": 0.6635401844978333,
|
|
"rewards/chosen": 0.6944223227211461,
|
|
"rewards/margins": 0.9343317835063383,
|
|
"rewards/rejected": -0.2399094607851921,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.328434944152832,
|
|
"kl": 20.705591201782227,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34441051.25552051,
|
|
"logits/rejected": -35800685.374613,
|
|
"logps/chosen": -487.750542192429,
|
|
"logps/rejected": -370.4547697368421,
|
|
"loss": 0.4644,
|
|
"loss/base_kto": 3.0393569469451904,
|
|
"loss/total_with_kl": 3.7152962684631348,
|
|
"metrics/advantage_var": 405.9695739746094,
|
|
"regularization/advantage_var": 405.9695739746094,
|
|
"regularization/kl": 0.6759393811225891,
|
|
"rewards/chosen": 0.7298758233007197,
|
|
"rewards/margins": 0.9655283099069987,
|
|
"rewards/rejected": -0.23565248660627902,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 15.256182670593262,
|
|
"kl": 21.343603134155273,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34241160.47819063,
|
|
"logits/rejected": -35653142.46293495,
|
|
"logps/chosen": -496.66892164781905,
|
|
"logps/rejected": -369.8054084720121,
|
|
"loss": 0.4604,
|
|
"loss/base_kto": 3.093393564224243,
|
|
"loss/total_with_kl": 3.68314790725708,
|
|
"metrics/advantage_var": 354.2069091796875,
|
|
"regularization/advantage_var": 354.2069091796875,
|
|
"regularization/kl": 0.5897545218467712,
|
|
"rewards/chosen": 0.7125130747362177,
|
|
"rewards/margins": 0.8750518195948599,
|
|
"rewards/rejected": -0.1625387448586422,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 15.188468933105469,
|
|
"kl": 19.63766860961914,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35101333.403278686,
|
|
"logits/rejected": -37411250.053731345,
|
|
"logps/chosen": -490.96331967213115,
|
|
"logps/rejected": -378.2164878731343,
|
|
"loss": 0.4626,
|
|
"loss/base_kto": 3.0533533096313477,
|
|
"loss/total_with_kl": 3.7009949684143066,
|
|
"metrics/advantage_var": 388.9740295410156,
|
|
"regularization/advantage_var": 388.9740295410156,
|
|
"regularization/kl": 0.6476417183876038,
|
|
"rewards/chosen": 0.6866738241226946,
|
|
"rewards/margins": 0.9182481303260528,
|
|
"rewards/rejected": -0.2315743062033582,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 8.538738250732422,
|
|
"kl": 16.847759246826172,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34908842.666666664,
|
|
"logits/rejected": -36901236.92867982,
|
|
"logps/chosen": -468.32573470209337,
|
|
"logps/rejected": -370.87239188163886,
|
|
"loss": 0.4619,
|
|
"loss/base_kto": 3.1275436878204346,
|
|
"loss/total_with_kl": 3.695369005203247,
|
|
"metrics/advantage_var": 341.0362854003906,
|
|
"regularization/advantage_var": 341.0362854003906,
|
|
"regularization/kl": 0.567825436592102,
|
|
"rewards/chosen": 0.6539842295377919,
|
|
"rewards/margins": 0.8895764594938096,
|
|
"rewards/rejected": -0.23559222995601764,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 12.581542015075684,
|
|
"kl": 18.04552459716797,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35341651.16957211,
|
|
"logits/rejected": -36038750.66872111,
|
|
"logps/chosen": -466.6060320919176,
|
|
"logps/rejected": -384.8725202234206,
|
|
"loss": 0.474,
|
|
"loss/base_kto": 3.106107473373413,
|
|
"loss/total_with_kl": 3.7923743724823,
|
|
"metrics/advantage_var": 412.1723327636719,
|
|
"regularization/advantage_var": 412.1723327636719,
|
|
"regularization/kl": 0.6862668991088867,
|
|
"rewards/chosen": 0.6502018609628814,
|
|
"rewards/margins": 0.870076614753268,
|
|
"rewards/rejected": -0.21987475379038665,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 12.283934593200684,
|
|
"kl": 18.97503089904785,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35337595.01298701,
|
|
"logits/rejected": -37089347.855421685,
|
|
"logps/chosen": -492.56960227272725,
|
|
"logps/rejected": -400.42469879518075,
|
|
"loss": 0.4606,
|
|
"loss/base_kto": 3.1100635528564453,
|
|
"loss/total_with_kl": 3.685138702392578,
|
|
"metrics/advantage_var": 345.39031982421875,
|
|
"regularization/advantage_var": 345.39031982421875,
|
|
"regularization/kl": 0.5750748515129089,
|
|
"rewards/chosen": 0.641826580097149,
|
|
"rewards/margins": 0.8806340744789731,
|
|
"rewards/rejected": -0.23880749438182416,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 22.152240753173828,
|
|
"kl": 18.58150291442871,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34276528.551724136,
|
|
"logits/rejected": -36221310.80373832,
|
|
"logps/chosen": -462.7879604231975,
|
|
"logps/rejected": -364.3088492990654,
|
|
"loss": 0.4625,
|
|
"loss/base_kto": 3.098907947540283,
|
|
"loss/total_with_kl": 3.6998727321624756,
|
|
"metrics/advantage_var": 360.9399719238281,
|
|
"regularization/advantage_var": 360.9399719238281,
|
|
"regularization/kl": 0.6009650826454163,
|
|
"rewards/chosen": 0.641452083767021,
|
|
"rewards/margins": 0.8731421548492104,
|
|
"rewards/rejected": -0.23169007108218945,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 11.47033405303955,
|
|
"kl": 18.98607635498047,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -35194479.925581396,
|
|
"logits/rejected": -36467191.130708665,
|
|
"logps/chosen": -492.0019864341085,
|
|
"logps/rejected": -392.89675196850396,
|
|
"loss": 0.464,
|
|
"loss/base_kto": 3.0971181392669678,
|
|
"loss/total_with_kl": 3.711954116821289,
|
|
"metrics/advantage_var": 369.27099609375,
|
|
"regularization/advantage_var": 369.27099609375,
|
|
"regularization/kl": 0.6148362159729004,
|
|
"rewards/chosen": 0.6864966104196948,
|
|
"rewards/margins": 0.9122782929522981,
|
|
"rewards/rejected": -0.22578168253260333,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 8.47675895690918,
|
|
"kl": 15.725268363952637,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34352238.32473445,
|
|
"logits/rejected": -34257004.83091787,
|
|
"logps/chosen": -468.64577010622156,
|
|
"logps/rejected": -342.00367351046697,
|
|
"loss": 0.452,
|
|
"loss/base_kto": 3.081899404525757,
|
|
"loss/total_with_kl": 3.6163601875305176,
|
|
"metrics/advantage_var": 320.9973449707031,
|
|
"regularization/advantage_var": 320.9973449707031,
|
|
"regularization/kl": 0.5344605445861816,
|
|
"rewards/chosen": 0.6775474721998292,
|
|
"rewards/margins": 0.9099925425249098,
|
|
"rewards/rejected": -0.23244507032508052,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_loss": 0.5005160395139652,
|
|
"train_runtime": 11075.1717,
|
|
"train_samples_per_second": 13.383,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|