2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 23.201366424560547,
|
||
|
|
"kl": 8.402630805969238,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37495531.68253968,
|
||
|
|
"logits/rejected": -37613577.45230769,
|
||
|
|
"logps/chosen": -493.8098214285714,
|
||
|
|
"logps/rejected": -370.74204326923075,
|
||
|
|
"loss": 0.6076,
|
||
|
|
"loss/base_kto": 3.9482905864715576,
|
||
|
|
"metrics/advantage_var": 196.1942901611328,
|
||
|
|
"regularization/lipschitz_norm": 932.10009765625,
|
||
|
|
"regularization/mmd": 0.12941110134124756,
|
||
|
|
"regularization/rkhs_norm": 154.0608367919922,
|
||
|
|
"regularization/w1": 0.7829639911651611,
|
||
|
|
"rewards/chosen": 0.003369693529038202,
|
||
|
|
"rewards/margins": 0.03324659031389397,
|
||
|
|
"rewards/rejected": -0.02987689678485577,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 21.66707420349121,
|
||
|
|
"kl": 4.654635906219482,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -36539791.72628305,
|
||
|
|
"logits/rejected": -37544074.24803767,
|
||
|
|
"logps/chosen": -501.1705870917574,
|
||
|
|
"logps/rejected": -377.8298910910518,
|
||
|
|
"loss": 0.5953,
|
||
|
|
"loss/base_kto": 3.900285482406616,
|
||
|
|
"metrics/advantage_var": 171.8822021484375,
|
||
|
|
"regularization/lipschitz_norm": 876.2872314453125,
|
||
|
|
"regularization/mmd": 0.12595663964748383,
|
||
|
|
"regularization/rkhs_norm": 149.9483642578125,
|
||
|
|
"regularization/w1": 0.7360813021659851,
|
||
|
|
"rewards/chosen": -0.012744417086730292,
|
||
|
|
"rewards/margins": 0.09687000457210751,
|
||
|
|
"rewards/rejected": -0.10961442165883781,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 23.52747344970703,
|
||
|
|
"kl": 16.981664657592773,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -36431273.88379205,
|
||
|
|
"logits/rejected": -37135350.18530352,
|
||
|
|
"logps/chosen": -472.2699732415902,
|
||
|
|
"logps/rejected": -394.61890974440894,
|
||
|
|
"loss": 0.5934,
|
||
|
|
"loss/base_kto": 3.80851149559021,
|
||
|
|
"metrics/advantage_var": 201.3900909423828,
|
||
|
|
"regularization/lipschitz_norm": 955.7496337890625,
|
||
|
|
"regularization/mmd": 0.136067196726799,
|
||
|
|
"regularization/rkhs_norm": 161.9847412109375,
|
||
|
|
"regularization/w1": 0.8028296828269958,
|
||
|
|
"rewards/chosen": 0.3549387753921182,
|
||
|
|
"rewards/margins": 0.16345281893806968,
|
||
|
|
"rewards/rejected": 0.19148595645404853,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 18.651674270629883,
|
||
|
|
"kl": 8.052300453186035,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -36878310.11690363,
|
||
|
|
"logits/rejected": -39378690.76970634,
|
||
|
|
"logps/chosen": -479.4358708530806,
|
||
|
|
"logps/rejected": -389.4826120556414,
|
||
|
|
"loss": 0.6024,
|
||
|
|
"loss/base_kto": 3.8863613605499268,
|
||
|
|
"metrics/advantage_var": 187.2263946533203,
|
||
|
|
"regularization/lipschitz_norm": 954.9240112304688,
|
||
|
|
"regularization/mmd": 0.1303941309452057,
|
||
|
|
"regularization/rkhs_norm": 155.23110961914062,
|
||
|
|
"regularization/w1": 0.8021361231803894,
|
||
|
|
"rewards/chosen": 0.09181685846940413,
|
||
|
|
"rewards/margins": 0.09587827290207716,
|
||
|
|
"rewards/rejected": -0.004061414432673034,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 17.754465103149414,
|
||
|
|
"kl": 3.9896461963653564,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -37180446.73617694,
|
||
|
|
"logits/rejected": -37164384.93972179,
|
||
|
|
"logps/chosen": -472.33180292259084,
|
||
|
|
"logps/rejected": -379.92250289799074,
|
||
|
|
"loss": 0.5984,
|
||
|
|
"loss/base_kto": 3.8640785217285156,
|
||
|
|
"metrics/advantage_var": 187.8689422607422,
|
||
|
|
"regularization/lipschitz_norm": 940.8318481445312,
|
||
|
|
"regularization/mmd": 0.13317380845546722,
|
||
|
|
"regularization/rkhs_norm": 158.54025268554688,
|
||
|
|
"regularization/w1": 0.7902988791465759,
|
||
|
|
"rewards/chosen": 0.05545770726497704,
|
||
|
|
"rewards/margins": 0.13958526017594836,
|
||
|
|
"rewards/rejected": -0.0841275529109713,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 24.01442527770996,
|
||
|
|
"kl": 12.528594970703125,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -37737079.10574018,
|
||
|
|
"logits/rejected": -38053033.00970874,
|
||
|
|
"logps/chosen": -476.94949018126886,
|
||
|
|
"logps/rejected": -366.1142799352751,
|
||
|
|
"loss": 0.5938,
|
||
|
|
"loss/base_kto": 3.8390586376190186,
|
||
|
|
"metrics/advantage_var": 192.7286834716797,
|
||
|
|
"regularization/lipschitz_norm": 926.2333984375,
|
||
|
|
"regularization/mmd": 0.1336306780576706,
|
||
|
|
"regularization/rkhs_norm": 159.08413696289062,
|
||
|
|
"regularization/w1": 0.7780360579490662,
|
||
|
|
"rewards/chosen": 0.22960575253581714,
|
||
|
|
"rewards/margins": 0.14638706381787936,
|
||
|
|
"rewards/rejected": 0.08321868871793778,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 18.925159454345703,
|
||
|
|
"kl": 11.071293830871582,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -36653491.2,
|
||
|
|
"logits/rejected": -38856371.2,
|
||
|
|
"logps/chosen": -497.14892578125,
|
||
|
|
"logps/rejected": -365.34912109375,
|
||
|
|
"loss": 0.5934,
|
||
|
|
"loss/base_kto": 3.84540057182312,
|
||
|
|
"metrics/advantage_var": 200.19244384765625,
|
||
|
|
"regularization/lipschitz_norm": 914.4345703125,
|
||
|
|
"regularization/mmd": 0.13373422622680664,
|
||
|
|
"regularization/rkhs_norm": 159.20742797851562,
|
||
|
|
"regularization/w1": 0.768125057220459,
|
||
|
|
"rewards/chosen": 0.17850465774536134,
|
||
|
|
"rewards/margins": 0.1355193853378296,
|
||
|
|
"rewards/rejected": 0.042985272407531736,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 18.5213565826416,
|
||
|
|
"kl": 8.384201049804688,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36425256.83435583,
|
||
|
|
"logits/rejected": -37792216.866242036,
|
||
|
|
"logps/chosen": -479.45115989263803,
|
||
|
|
"logps/rejected": -365.8758210589172,
|
||
|
|
"loss": 0.5931,
|
||
|
|
"loss/base_kto": 3.806594133377075,
|
||
|
|
"metrics/advantage_var": 225.21328735351562,
|
||
|
|
"regularization/lipschitz_norm": 953.2193603515625,
|
||
|
|
"regularization/mmd": 0.1376592069864273,
|
||
|
|
"regularization/rkhs_norm": 163.8800048828125,
|
||
|
|
"regularization/w1": 0.800704300403595,
|
||
|
|
"rewards/chosen": 0.13271222962923576,
|
||
|
|
"rewards/margins": 0.17763238558572275,
|
||
|
|
"rewards/rejected": -0.04492015595648699,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 25.310150146484375,
|
||
|
|
"kl": 4.375081539154053,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -36966065.294498384,
|
||
|
|
"logits/rejected": -36937697.06344411,
|
||
|
|
"logps/chosen": -470.877427184466,
|
||
|
|
"logps/rejected": -370.95548527190334,
|
||
|
|
"loss": 0.5955,
|
||
|
|
"loss/base_kto": 3.823848009109497,
|
||
|
|
"metrics/advantage_var": 225.6634063720703,
|
||
|
|
"regularization/lipschitz_norm": 953.9823608398438,
|
||
|
|
"regularization/mmd": 0.13904273509979248,
|
||
|
|
"regularization/rkhs_norm": 165.52708435058594,
|
||
|
|
"regularization/w1": 0.8013450503349304,
|
||
|
|
"rewards/chosen": 0.031231395635018457,
|
||
|
|
"rewards/margins": 0.16485980290878638,
|
||
|
|
"rewards/rejected": -0.13362840727376793,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 23.374717712402344,
|
||
|
|
"kl": 8.640753746032715,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -36372073.06840391,
|
||
|
|
"logits/rejected": -37955116.58858859,
|
||
|
|
"logps/chosen": -473.8562703583062,
|
||
|
|
"logps/rejected": -369.1987378003003,
|
||
|
|
"loss": 0.5867,
|
||
|
|
"loss/base_kto": 3.786153554916382,
|
||
|
|
"metrics/advantage_var": 191.48971557617188,
|
||
|
|
"regularization/lipschitz_norm": 921.3744506835938,
|
||
|
|
"regularization/mmd": 0.13376352190971375,
|
||
|
|
"regularization/rkhs_norm": 159.24229431152344,
|
||
|
|
"regularization/w1": 0.773954451084137,
|
||
|
|
"rewards/chosen": 0.0762817432515396,
|
||
|
|
"rewards/margins": 0.17907845246685844,
|
||
|
|
"rewards/rejected": -0.10279670921531883,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 19.559431076049805,
|
||
|
|
"kl": 11.47446346282959,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -36434969.6377025,
|
||
|
|
"logits/rejected": -37157310.40266223,
|
||
|
|
"logps/chosen": -494.7851620029455,
|
||
|
|
"logps/rejected": -380.09073419301166,
|
||
|
|
"loss": 0.5984,
|
||
|
|
"loss/base_kto": 3.7972962856292725,
|
||
|
|
"metrics/advantage_var": 224.0395965576172,
|
||
|
|
"regularization/lipschitz_norm": 1006.0599975585938,
|
||
|
|
"regularization/mmd": 0.14461135864257812,
|
||
|
|
"regularization/rkhs_norm": 172.1563720703125,
|
||
|
|
"regularization/w1": 0.845090389251709,
|
||
|
|
"rewards/chosen": 0.15154651759826032,
|
||
|
|
"rewards/margins": 0.19963302673159766,
|
||
|
|
"rewards/rejected": -0.04808650913333734,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 21.561067581176758,
|
||
|
|
"kl": 10.763829231262207,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -35681289.721518986,
|
||
|
|
"logits/rejected": -37185864.69135802,
|
||
|
|
"logps/chosen": -497.31101661392404,
|
||
|
|
"logps/rejected": -376.658203125,
|
||
|
|
"loss": 0.6012,
|
||
|
|
"loss/base_kto": 3.8065345287323,
|
||
|
|
"metrics/advantage_var": 228.43467712402344,
|
||
|
|
"regularization/lipschitz_norm": 1020.8681640625,
|
||
|
|
"regularization/mmd": 0.1455560326576233,
|
||
|
|
"regularization/rkhs_norm": 173.28099060058594,
|
||
|
|
"regularization/w1": 0.8575292825698853,
|
||
|
|
"rewards/chosen": 0.2424291296850277,
|
||
|
|
"rewards/margins": 0.19370460491624544,
|
||
|
|
"rewards/rejected": 0.04872452476878225,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 20.845365524291992,
|
||
|
|
"kl": 9.152388572692871,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -36734544.192771085,
|
||
|
|
"logits/rejected": -37080991.584415585,
|
||
|
|
"logps/chosen": -481.0437688253012,
|
||
|
|
"logps/rejected": -363.7502536525974,
|
||
|
|
"loss": 0.5893,
|
||
|
|
"loss/base_kto": 3.7788803577423096,
|
||
|
|
"metrics/advantage_var": 207.372314453125,
|
||
|
|
"regularization/lipschitz_norm": 950.1049194335938,
|
||
|
|
"regularization/mmd": 0.13745544850826263,
|
||
|
|
"regularization/rkhs_norm": 163.63742065429688,
|
||
|
|
"regularization/w1": 0.7980880737304688,
|
||
|
|
"rewards/chosen": 0.1595219416790698,
|
||
|
|
"rewards/margins": 0.20397445409813877,
|
||
|
|
"rewards/rejected": -0.04445251241906897,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 22.008657455444336,
|
||
|
|
"kl": 23.633359909057617,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -36656650.96024465,
|
||
|
|
"logits/rejected": -38405116.7284345,
|
||
|
|
"logps/chosen": -526.9809824159022,
|
||
|
|
"logps/rejected": -366.6975339456869,
|
||
|
|
"loss": 0.5981,
|
||
|
|
"loss/base_kto": 3.7402045726776123,
|
||
|
|
"metrics/advantage_var": 257.9858093261719,
|
||
|
|
"regularization/lipschitz_norm": 1060.8779296875,
|
||
|
|
"regularization/mmd": 0.15345418453216553,
|
||
|
|
"regularization/rkhs_norm": 182.6835479736328,
|
||
|
|
"regularization/w1": 0.8911373019218445,
|
||
|
|
"rewards/chosen": 0.4148859613167766,
|
||
|
|
"rewards/margins": 0.24334470467115957,
|
||
|
|
"rewards/rejected": 0.171541256645617,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 19.194862365722656,
|
||
|
|
"kl": 18.423542022705078,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -36171171.77639752,
|
||
|
|
"logits/rejected": -37756924.77987421,
|
||
|
|
"logps/chosen": -478.0741459627329,
|
||
|
|
"logps/rejected": -355.2376179245283,
|
||
|
|
"loss": 0.5877,
|
||
|
|
"loss/base_kto": 3.7170631885528564,
|
||
|
|
"metrics/advantage_var": 219.2563018798828,
|
||
|
|
"regularization/lipschitz_norm": 1003.7697143554688,
|
||
|
|
"regularization/mmd": 0.14144547283649445,
|
||
|
|
"regularization/rkhs_norm": 168.387451171875,
|
||
|
|
"regularization/w1": 0.8431665301322937,
|
||
|
|
"rewards/chosen": 0.31452728650584727,
|
||
|
|
"rewards/margins": 0.26064615112910294,
|
||
|
|
"rewards/rejected": 0.0538811353767443,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 19.775278091430664,
|
||
|
|
"kl": 14.477513313293457,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -36717425.913875595,
|
||
|
|
"logits/rejected": -37830612.091883615,
|
||
|
|
"logps/chosen": -464.4652611642743,
|
||
|
|
"logps/rejected": -382.3154192189893,
|
||
|
|
"loss": 0.5956,
|
||
|
|
"loss/base_kto": 3.818542957305908,
|
||
|
|
"metrics/advantage_var": 218.4783172607422,
|
||
|
|
"regularization/lipschitz_norm": 959.4794921875,
|
||
|
|
"regularization/mmd": 0.1400190144777298,
|
||
|
|
"regularization/rkhs_norm": 166.68931579589844,
|
||
|
|
"regularization/w1": 0.8059626817703247,
|
||
|
|
"rewards/chosen": 0.24981453392113986,
|
||
|
|
"rewards/margins": 0.1744485180998553,
|
||
|
|
"rewards/rejected": 0.07536601582128458,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 18.630006790161133,
|
||
|
|
"kl": 12.413385391235352,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -36214699.96352584,
|
||
|
|
"logits/rejected": -38260676.73311897,
|
||
|
|
"logps/chosen": -469.5349069148936,
|
||
|
|
"logps/rejected": -368.364700562701,
|
||
|
|
"loss": 0.5991,
|
||
|
|
"loss/base_kto": 3.7885711193084717,
|
||
|
|
"metrics/advantage_var": 247.23081970214844,
|
||
|
|
"regularization/lipschitz_norm": 1021.1866455078125,
|
||
|
|
"regularization/mmd": 0.14620965719223022,
|
||
|
|
"regularization/rkhs_norm": 174.0591278076172,
|
||
|
|
"regularization/w1": 0.8577967882156372,
|
||
|
|
"rewards/chosen": 0.2249682116291081,
|
||
|
|
"rewards/margins": 0.20342048321632047,
|
||
|
|
"rewards/rejected": 0.02154772841278763,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 18.37293815612793,
|
||
|
|
"kl": 8.929717063903809,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -36370652.03305785,
|
||
|
|
"logits/rejected": -38313492.48,
|
||
|
|
"logps/chosen": -482.0063533057851,
|
||
|
|
"logps/rejected": -373.780625,
|
||
|
|
"loss": 0.5982,
|
||
|
|
"loss/base_kto": 3.784968614578247,
|
||
|
|
"metrics/advantage_var": 242.4626007080078,
|
||
|
|
"regularization/lipschitz_norm": 1016.2395629882812,
|
||
|
|
"regularization/mmd": 0.14667455852031708,
|
||
|
|
"regularization/rkhs_norm": 174.61256408691406,
|
||
|
|
"regularization/w1": 0.8536412119865417,
|
||
|
|
"rewards/chosen": 0.11054369713649277,
|
||
|
|
"rewards/margins": 0.1864359746068342,
|
||
|
|
"rewards/rejected": -0.07589227747034144,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 25.06922721862793,
|
||
|
|
"kl": 10.867541313171387,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -35887252.593059935,
|
||
|
|
"logits/rejected": -38262558.91021672,
|
||
|
|
"logps/chosen": -493.1877957413249,
|
||
|
|
"logps/rejected": -358.5596458978328,
|
||
|
|
"loss": 0.5944,
|
||
|
|
"loss/base_kto": 3.784297227859497,
|
||
|
|
"metrics/advantage_var": 221.9957275390625,
|
||
|
|
"regularization/lipschitz_norm": 986.2423706054688,
|
||
|
|
"regularization/mmd": 0.14268994331359863,
|
||
|
|
"regularization/rkhs_norm": 169.8689727783203,
|
||
|
|
"regularization/w1": 0.8284436464309692,
|
||
|
|
"rewards/chosen": 0.15735308529826744,
|
||
|
|
"rewards/margins": 0.19731536372931247,
|
||
|
|
"rewards/rejected": -0.039962278431045016,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 19.531946182250977,
|
||
|
|
"kl": 8.208697319030762,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -34904213.93146417,
|
||
|
|
"logits/rejected": -36032232.72727273,
|
||
|
|
"logps/chosen": -510.03767523364485,
|
||
|
|
"logps/rejected": -383.8157327586207,
|
||
|
|
"loss": 0.6017,
|
||
|
|
"loss/base_kto": 3.784752607345581,
|
||
|
|
"metrics/advantage_var": 243.6321258544922,
|
||
|
|
"regularization/lipschitz_norm": 1048.397705078125,
|
||
|
|
"regularization/mmd": 0.148299902677536,
|
||
|
|
"regularization/rkhs_norm": 176.54750061035156,
|
||
|
|
"regularization/w1": 0.8806540369987488,
|
||
|
|
"rewards/chosen": 0.041582713617342656,
|
||
|
|
"rewards/margins": 0.19421676327508658,
|
||
|
|
"rewards/rejected": -0.15263404965774394,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 22.28571891784668,
|
||
|
|
"kl": 7.1474609375,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36555004.26580227,
|
||
|
|
"logits/rejected": -37975205.26093514,
|
||
|
|
"logps/chosen": -466.6588837115073,
|
||
|
|
"logps/rejected": -379.2359068627451,
|
||
|
|
"loss": 0.6035,
|
||
|
|
"loss/base_kto": 3.7734246253967285,
|
||
|
|
"metrics/advantage_var": 280.4195861816406,
|
||
|
|
"regularization/lipschitz_norm": 1074.358642578125,
|
||
|
|
"regularization/mmd": 0.1519424468278885,
|
||
|
|
"regularization/rkhs_norm": 180.8838653564453,
|
||
|
|
"regularization/w1": 0.9024612307548523,
|
||
|
|
"rewards/chosen": 0.03843623428715686,
|
||
|
|
"rewards/margins": 0.20931593776860954,
|
||
|
|
"rewards/rejected": -0.17087970348145268,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 25.39838409423828,
|
||
|
|
"kl": 11.798993110656738,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -37201131.83030303,
|
||
|
|
"logits/rejected": -37188317.31612903,
|
||
|
|
"logps/chosen": -480.7503787878788,
|
||
|
|
"logps/rejected": -400.02147177419357,
|
||
|
|
"loss": 0.6058,
|
||
|
|
"loss/base_kto": 3.740493059158325,
|
||
|
|
"metrics/advantage_var": 290.8243713378906,
|
||
|
|
"regularization/lipschitz_norm": 1123.6910400390625,
|
||
|
|
"regularization/mmd": 0.1618811935186386,
|
||
|
|
"regularization/rkhs_norm": 192.71571350097656,
|
||
|
|
"regularization/w1": 0.9439005255699158,
|
||
|
|
"rewards/chosen": 0.1848052978515625,
|
||
|
|
"rewards/margins": 0.24894289816579512,
|
||
|
|
"rewards/rejected": -0.06413760031423262,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 18.92000961303711,
|
||
|
|
"kl": 6.131280422210693,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -36843587.70247934,
|
||
|
|
"logits/rejected": -37389167.88148148,
|
||
|
|
"logps/chosen": -482.91239669421486,
|
||
|
|
"logps/rejected": -383.0197685185185,
|
||
|
|
"loss": 0.5967,
|
||
|
|
"loss/base_kto": 3.7245500087738037,
|
||
|
|
"metrics/advantage_var": 257.1286315917969,
|
||
|
|
"regularization/lipschitz_norm": 1065.8896484375,
|
||
|
|
"regularization/mmd": 0.15357382595539093,
|
||
|
|
"regularization/rkhs_norm": 182.8260040283203,
|
||
|
|
"regularization/w1": 0.8953474164009094,
|
||
|
|
"rewards/chosen": -0.01881012561892675,
|
||
|
|
"rewards/margins": 0.2506267685506334,
|
||
|
|
"rewards/rejected": -0.2694368941695602,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 24.400739669799805,
|
||
|
|
"kl": 2.4771485328674316,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -36257142.4,
|
||
|
|
"logits/rejected": -37287686.4,
|
||
|
|
"logps/chosen": -513.2826171875,
|
||
|
|
"logps/rejected": -378.8910888671875,
|
||
|
|
"loss": 0.6058,
|
||
|
|
"loss/base_kto": 3.782191038131714,
|
||
|
|
"metrics/advantage_var": 264.2885437011719,
|
||
|
|
"regularization/lipschitz_norm": 1080.640625,
|
||
|
|
"regularization/mmd": 0.15670809149742126,
|
||
|
|
"regularization/rkhs_norm": 186.5572509765625,
|
||
|
|
"regularization/w1": 0.9077381491661072,
|
||
|
|
"rewards/chosen": -0.1675581932067871,
|
||
|
|
"rewards/margins": 0.2288973331451416,
|
||
|
|
"rewards/rejected": -0.3964555263519287,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 17.90960693359375,
|
||
|
|
"kl": 3.5062294006347656,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36471070.98089172,
|
||
|
|
"logits/rejected": -36497087.607361965,
|
||
|
|
"logps/chosen": -517.266520700637,
|
||
|
|
"logps/rejected": -397.68009490030676,
|
||
|
|
"loss": 0.602,
|
||
|
|
"loss/base_kto": 3.7644765377044678,
|
||
|
|
"metrics/advantage_var": 252.4976043701172,
|
||
|
|
"regularization/lipschitz_norm": 1070.3653564453125,
|
||
|
|
"regularization/mmd": 0.15278176963329315,
|
||
|
|
"regularization/rkhs_norm": 181.883056640625,
|
||
|
|
"regularization/w1": 0.8991069197654724,
|
||
|
|
"rewards/chosen": -0.0013660291197952952,
|
||
|
|
"rewards/margins": 0.2330090176863061,
|
||
|
|
"rewards/rejected": -0.2343750468061014,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 17.765396118164062,
|
||
|
|
"kl": 9.653647422790527,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -36413656.73619632,
|
||
|
|
"logits/rejected": -37935143.133757964,
|
||
|
|
"logps/chosen": -490.32199003067484,
|
||
|
|
"logps/rejected": -368.073721138535,
|
||
|
|
"loss": 0.5972,
|
||
|
|
"loss/base_kto": 3.723569631576538,
|
||
|
|
"metrics/advantage_var": 252.1516571044922,
|
||
|
|
"regularization/lipschitz_norm": 1072.605712890625,
|
||
|
|
"regularization/mmd": 0.15296387672424316,
|
||
|
|
"regularization/rkhs_norm": 182.099853515625,
|
||
|
|
"regularization/w1": 0.900988757610321,
|
||
|
|
"rewards/chosen": 0.19025890373744847,
|
||
|
|
"rewards/margins": 0.25742432101381874,
|
||
|
|
"rewards/rejected": -0.06716541727637029,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 17.09473991394043,
|
||
|
|
"kl": 17.791292190551758,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -34637766.607669614,
|
||
|
|
"logits/rejected": -37390295.176079735,
|
||
|
|
"logps/chosen": -510.04747418879055,
|
||
|
|
"logps/rejected": -348.90391403654485,
|
||
|
|
"loss": 0.5927,
|
||
|
|
"loss/base_kto": 3.67045259475708,
|
||
|
|
"metrics/advantage_var": 263.37255859375,
|
||
|
|
"regularization/lipschitz_norm": 1087.25341796875,
|
||
|
|
"regularization/mmd": 0.15750935673713684,
|
||
|
|
"regularization/rkhs_norm": 187.51113891601562,
|
||
|
|
"regularization/w1": 0.9132928848266602,
|
||
|
|
"rewards/chosen": 0.3956253366836053,
|
||
|
|
"rewards/margins": 0.29386015079201344,
|
||
|
|
"rewards/rejected": 0.10176518589159184,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 21.598224639892578,
|
||
|
|
"kl": 16.767559051513672,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -34827741.219589256,
|
||
|
|
"logits/rejected": -37523544.630602784,
|
||
|
|
"logps/chosen": -485.0288309636651,
|
||
|
|
"logps/rejected": -346.01905428902626,
|
||
|
|
"loss": 0.5954,
|
||
|
|
"loss/base_kto": 3.773083209991455,
|
||
|
|
"metrics/advantage_var": 223.81289672851562,
|
||
|
|
"regularization/lipschitz_norm": 1007.4150390625,
|
||
|
|
"regularization/mmd": 0.14403896033763885,
|
||
|
|
"regularization/rkhs_norm": 171.47496032714844,
|
||
|
|
"regularization/w1": 0.8462285995483398,
|
||
|
|
"rewards/chosen": 0.23780579288235584,
|
||
|
|
"rewards/margins": 0.17529541596243697,
|
||
|
|
"rewards/rejected": 0.06251037691991886,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 24.668848037719727,
|
||
|
|
"kl": 14.342513084411621,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36728429.48427673,
|
||
|
|
"logits/rejected": -38351941.96273292,
|
||
|
|
"logps/chosen": -479.85225039308176,
|
||
|
|
"logps/rejected": -387.6481948757764,
|
||
|
|
"loss": 0.5875,
|
||
|
|
"loss/base_kto": 3.7165493965148926,
|
||
|
|
"metrics/advantage_var": 223.0778350830078,
|
||
|
|
"regularization/lipschitz_norm": 1000.13037109375,
|
||
|
|
"regularization/mmd": 0.14316968619823456,
|
||
|
|
"regularization/rkhs_norm": 170.4401092529297,
|
||
|
|
"regularization/w1": 0.8401094675064087,
|
||
|
|
"rewards/chosen": 0.23220103041930767,
|
||
|
|
"rewards/margins": 0.2777852381554948,
|
||
|
|
"rewards/rejected": -0.0455842077361871,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 17.468168258666992,
|
||
|
|
"kl": 10.173858642578125,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -37258064.5482866,
|
||
|
|
"logits/rejected": -38367373.24137931,
|
||
|
|
"logps/chosen": -481.41442757009344,
|
||
|
|
"logps/rejected": -366.62402037617557,
|
||
|
|
"loss": 0.594,
|
||
|
|
"loss/base_kto": 3.7690837383270264,
|
||
|
|
"metrics/advantage_var": 238.7223358154297,
|
||
|
|
"regularization/lipschitz_norm": 993.779296875,
|
||
|
|
"regularization/mmd": 0.14850184321403503,
|
||
|
|
"regularization/rkhs_norm": 176.7878875732422,
|
||
|
|
"regularization/w1": 0.8347746133804321,
|
||
|
|
"rewards/chosen": 0.1052568619867723,
|
||
|
|
"rewards/margins": 0.22628912241517404,
|
||
|
|
"rewards/rejected": -0.12103226042840175,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 20.574800491333008,
|
||
|
|
"kl": 15.027544975280762,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36774962.7904,
|
||
|
|
"logits/rejected": -36964917.93587786,
|
||
|
|
"logps/chosen": -494.93555,
|
||
|
|
"logps/rejected": -365.2274809160305,
|
||
|
|
"loss": 0.5893,
|
||
|
|
"loss/base_kto": 3.7046680450439453,
|
||
|
|
"metrics/advantage_var": 255.10940551757812,
|
||
|
|
"regularization/lipschitz_norm": 1021.0386962890625,
|
||
|
|
"regularization/mmd": 0.15168698132038116,
|
||
|
|
"regularization/rkhs_norm": 180.57974243164062,
|
||
|
|
"regularization/w1": 0.857672393321991,
|
||
|
|
"rewards/chosen": 0.2168187255859375,
|
||
|
|
"rewards/margins": 0.27910208572504175,
|
||
|
|
"rewards/rejected": -0.062283360139104245,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 17.16766929626465,
|
||
|
|
"kl": 8.609379768371582,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -37099874.645865835,
|
||
|
|
"logits/rejected": -36729216.60093897,
|
||
|
|
"logps/chosen": -463.9157566302652,
|
||
|
|
"logps/rejected": -387.1952269170579,
|
||
|
|
"loss": 0.5918,
|
||
|
|
"loss/base_kto": 3.7824878692626953,
|
||
|
|
"metrics/advantage_var": 199.96597290039062,
|
||
|
|
"regularization/lipschitz_norm": 968.4484252929688,
|
||
|
|
"regularization/mmd": 0.13863173127174377,
|
||
|
|
"regularization/rkhs_norm": 165.0377655029297,
|
||
|
|
"regularization/w1": 0.8134967684745789,
|
||
|
|
"rewards/chosen": 0.055951039616291685,
|
||
|
|
"rewards/margins": 0.1907830367668382,
|
||
|
|
"rewards/rejected": -0.1348319971505465,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 19.824546813964844,
|
||
|
|
"kl": 13.744544982910156,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36412412.81987578,
|
||
|
|
"logits/rejected": -36773894.440251574,
|
||
|
|
"logps/chosen": -476.6167993012422,
|
||
|
|
"logps/rejected": -381.044147995283,
|
||
|
|
"loss": 0.5955,
|
||
|
|
"loss/base_kto": 3.731142044067383,
|
||
|
|
"metrics/advantage_var": 249.41421508789062,
|
||
|
|
"regularization/lipschitz_norm": 1050.92724609375,
|
||
|
|
"regularization/mmd": 0.15047474205493927,
|
||
|
|
"regularization/rkhs_norm": 179.1365966796875,
|
||
|
|
"regularization/w1": 0.8827788233757019,
|
||
|
|
"rewards/chosen": 0.16196018716563348,
|
||
|
|
"rewards/margins": 0.24281847773597054,
|
||
|
|
"rewards/rejected": -0.08085829057033707,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 16.35430335998535,
|
||
|
|
"kl": 10.43152904510498,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -37789531.36977492,
|
||
|
|
"logits/rejected": -38725912.12158055,
|
||
|
|
"logps/chosen": -486.65770699356915,
|
||
|
|
"logps/rejected": -367.61880224164133,
|
||
|
|
"loss": 0.594,
|
||
|
|
"loss/base_kto": 3.7184975147247314,
|
||
|
|
"metrics/advantage_var": 277.5556640625,
|
||
|
|
"regularization/lipschitz_norm": 1048.2724609375,
|
||
|
|
"regularization/mmd": 0.1530371755361557,
|
||
|
|
"regularization/rkhs_norm": 182.18710327148438,
|
||
|
|
"regularization/w1": 0.880548894405365,
|
||
|
|
"rewards/chosen": 0.10123903575050869,
|
||
|
|
"rewards/margins": 0.24801486985435675,
|
||
|
|
"rewards/rejected": -0.14677583410384806,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 18.478219985961914,
|
||
|
|
"kl": 13.821173667907715,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -37496069.8805513,
|
||
|
|
"logits/rejected": -37963668.21052632,
|
||
|
|
"logps/chosen": -480.70568529862175,
|
||
|
|
"logps/rejected": -371.37509968102074,
|
||
|
|
"loss": 0.5924,
|
||
|
|
"loss/base_kto": 3.7181453704833984,
|
||
|
|
"metrics/advantage_var": 249.6005859375,
|
||
|
|
"regularization/lipschitz_norm": 1037.6195068359375,
|
||
|
|
"regularization/mmd": 0.14965471625328064,
|
||
|
|
"regularization/rkhs_norm": 178.1603546142578,
|
||
|
|
"regularization/w1": 0.8716004490852356,
|
||
|
|
"rewards/chosen": 0.17842933923508086,
|
||
|
|
"rewards/margins": 0.26581969741312983,
|
||
|
|
"rewards/rejected": -0.087390358178049,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 15.74271011352539,
|
||
|
|
"kl": 22.262348175048828,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -35860863.21472393,
|
||
|
|
"logits/rejected": -36158503.133757964,
|
||
|
|
"logps/chosen": -467.2310199386503,
|
||
|
|
"logps/rejected": -380.0896198248408,
|
||
|
|
"loss": 0.5932,
|
||
|
|
"loss/base_kto": 3.7683913707733154,
|
||
|
|
"metrics/advantage_var": 235.16421508789062,
|
||
|
|
"regularization/lipschitz_norm": 992.5671997070312,
|
||
|
|
"regularization/mmd": 0.14327631890773773,
|
||
|
|
"regularization/rkhs_norm": 170.56704711914062,
|
||
|
|
"regularization/w1": 0.8337564468383789,
|
||
|
|
"rewards/chosen": 0.32669114188913917,
|
||
|
|
"rewards/margins": 0.18486287217361744,
|
||
|
|
"rewards/rejected": 0.14182826971552173,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 15.067955017089844,
|
||
|
|
"kl": 16.901716232299805,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36223467.003154576,
|
||
|
|
"logits/rejected": -38478001.53560372,
|
||
|
|
"logps/chosen": -500.4433162460568,
|
||
|
|
"logps/rejected": -345.9794649767802,
|
||
|
|
"loss": 0.5942,
|
||
|
|
"loss/base_kto": 3.7538509368896484,
|
||
|
|
"metrics/advantage_var": 227.28067016601562,
|
||
|
|
"regularization/lipschitz_norm": 1016.8820190429688,
|
||
|
|
"regularization/mmd": 0.14528994262218475,
|
||
|
|
"regularization/rkhs_norm": 172.96421813964844,
|
||
|
|
"regularization/w1": 0.8541808128356934,
|
||
|
|
"rewards/chosen": 0.25420232826979,
|
||
|
|
"rewards/margins": 0.20404441515626387,
|
||
|
|
"rewards/rejected": 0.050157913113526145,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 21.448060989379883,
|
||
|
|
"kl": 12.525344848632812,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -35428295.4601227,
|
||
|
|
"logits/rejected": -36551993.070063695,
|
||
|
|
"logps/chosen": -466.9871549079755,
|
||
|
|
"logps/rejected": -373.3005573248408,
|
||
|
|
"loss": 0.5967,
|
||
|
|
"loss/base_kto": 3.7189204692840576,
|
||
|
|
"metrics/advantage_var": 253.89907836914062,
|
||
|
|
"regularization/lipschitz_norm": 1074.1412353515625,
|
||
|
|
"regularization/mmd": 0.1524512618780136,
|
||
|
|
"regularization/rkhs_norm": 181.48960876464844,
|
||
|
|
"regularization/w1": 0.90227872133255,
|
||
|
|
"rewards/chosen": 0.1976150793531921,
|
||
|
|
"rewards/margins": 0.2684409173328429,
|
||
|
|
"rewards/rejected": -0.07082583797965081,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 21.17198944091797,
|
||
|
|
"kl": 10.957121849060059,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -35724504.01288245,
|
||
|
|
"logits/rejected": -37041106.80060883,
|
||
|
|
"logps/chosen": -442.47997181964575,
|
||
|
|
"logps/rejected": -350.06259512937595,
|
||
|
|
"loss": 0.6079,
|
||
|
|
"loss/base_kto": 3.5812816619873047,
|
||
|
|
"metrics/advantage_var": 738.0580444335938,
|
||
|
|
"regularization/lipschitz_norm": 1302.969482421875,
|
||
|
|
"regularization/mmd": 0.18771572411060333,
|
||
|
|
"regularization/rkhs_norm": 223.47109985351562,
|
||
|
|
"regularization/w1": 1.094494342803955,
|
||
|
|
"rewards/chosen": 0.21747496968882096,
|
||
|
|
"rewards/margins": 0.38237252259710436,
|
||
|
|
"rewards/rejected": -0.1648975529082834,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 16.01467514038086,
|
||
|
|
"kl": 7.424740791320801,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -38050041.836276084,
|
||
|
|
"logits/rejected": -37941416.32876712,
|
||
|
|
"logps/chosen": -479.805176565008,
|
||
|
|
"logps/rejected": -375.0091324200913,
|
||
|
|
"loss": 0.6327,
|
||
|
|
"loss/base_kto": 3.334271192550659,
|
||
|
|
"metrics/advantage_var": 783.7450561523438,
|
||
|
|
"regularization/lipschitz_norm": 1760.3284912109375,
|
||
|
|
"regularization/mmd": 0.24859409034252167,
|
||
|
|
"regularization/rkhs_norm": 295.94537353515625,
|
||
|
|
"regularization/w1": 1.4786758422851562,
|
||
|
|
"rewards/chosen": 0.3345975676853431,
|
||
|
|
"rewards/margins": 0.7775446070064761,
|
||
|
|
"rewards/rejected": -0.442947039321133,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 15.23444652557373,
|
||
|
|
"kl": 8.888628959655762,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -36096753.50943396,
|
||
|
|
"logits/rejected": -37404125.01863354,
|
||
|
|
"logps/chosen": -482.1945754716981,
|
||
|
|
"logps/rejected": -374.61558618012424,
|
||
|
|
"loss": 0.596,
|
||
|
|
"loss/base_kto": 3.2763848304748535,
|
||
|
|
"metrics/advantage_var": 552.4193725585938,
|
||
|
|
"regularization/lipschitz_norm": 1517.012939453125,
|
||
|
|
"regularization/mmd": 0.217189222574234,
|
||
|
|
"regularization/rkhs_norm": 258.55859375,
|
||
|
|
"regularization/w1": 1.274290919303894,
|
||
|
|
"rewards/chosen": 0.33045047784001574,
|
||
|
|
"rewards/margins": 0.784143275388298,
|
||
|
|
"rewards/rejected": -0.45369279754828223,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 19.889047622680664,
|
||
|
|
"kl": 11.319856643676758,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -36004028.75576037,
|
||
|
|
"logits/rejected": -37463611.42130366,
|
||
|
|
"logps/chosen": -483.4574692780338,
|
||
|
|
"logps/rejected": -370.69569753577105,
|
||
|
|
"loss": 0.6181,
|
||
|
|
"loss/base_kto": 3.2467949390411377,
|
||
|
|
"metrics/advantage_var": 748.3344116210938,
|
||
|
|
"regularization/lipschitz_norm": 1723.995361328125,
|
||
|
|
"regularization/mmd": 0.24950304627418518,
|
||
|
|
"regularization/rkhs_norm": 297.0274353027344,
|
||
|
|
"regularization/w1": 1.4481559991836548,
|
||
|
|
"rewards/chosen": 0.548060779014857,
|
||
|
|
"rewards/margins": 0.8650489977741723,
|
||
|
|
"rewards/rejected": -0.3169882187593154,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 22.57478904724121,
|
||
|
|
"kl": 15.289138793945312,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -36593874.542056076,
|
||
|
|
"logits/rejected": -36750891.335423194,
|
||
|
|
"logps/chosen": -478.58771417445485,
|
||
|
|
"logps/rejected": -387.89851097178683,
|
||
|
|
"loss": 0.6073,
|
||
|
|
"loss/base_kto": 3.307131290435791,
|
||
|
|
"metrics/advantage_var": 580.6983032226562,
|
||
|
|
"regularization/lipschitz_norm": 1584.091552734375,
|
||
|
|
"regularization/mmd": 0.22041180729866028,
|
||
|
|
"regularization/rkhs_norm": 262.39501953125,
|
||
|
|
"regularization/w1": 1.3306368589401245,
|
||
|
|
"rewards/chosen": 0.46975727021879865,
|
||
|
|
"rewards/margins": 0.7718949022001491,
|
||
|
|
"rewards/rejected": -0.3021376319813504,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 19.370643615722656,
|
||
|
|
"kl": 21.642431259155273,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -35021520.9470405,
|
||
|
|
"logits/rejected": -37680757.1661442,
|
||
|
|
"logps/chosen": -464.11268496884736,
|
||
|
|
"logps/rejected": -375.8528115203762,
|
||
|
|
"loss": 0.6114,
|
||
|
|
"loss/base_kto": 3.313422441482544,
|
||
|
|
"metrics/advantage_var": 648.5211791992188,
|
||
|
|
"regularization/lipschitz_norm": 1606.2928466796875,
|
||
|
|
"regularization/mmd": 0.2287592887878418,
|
||
|
|
"regularization/rkhs_norm": 272.3324890136719,
|
||
|
|
"regularization/w1": 1.3492859601974487,
|
||
|
|
"rewards/chosen": 0.6054001567519713,
|
||
|
|
"rewards/margins": 0.7418271840566895,
|
||
|
|
"rewards/rejected": -0.13642702730471812,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 19.645137786865234,
|
||
|
|
"kl": 13.119185447692871,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -34872693.349470496,
|
||
|
|
"logits/rejected": -35669575.96122779,
|
||
|
|
"logps/chosen": -485.15951210287443,
|
||
|
|
"logps/rejected": -367.42808461227787,
|
||
|
|
"loss": 0.6179,
|
||
|
|
"loss/base_kto": 3.326477527618408,
|
||
|
|
"metrics/advantage_var": 641.0195922851562,
|
||
|
|
"regularization/lipschitz_norm": 1654.0477294921875,
|
||
|
|
"regularization/mmd": 0.22714853286743164,
|
||
|
|
"regularization/rkhs_norm": 270.4148864746094,
|
||
|
|
"regularization/w1": 1.3894000053405762,
|
||
|
|
"rewards/chosen": 0.4846454106733406,
|
||
|
|
"rewards/margins": 0.7433561578698648,
|
||
|
|
"rewards/rejected": -0.25871074719652415,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 20.28074073791504,
|
||
|
|
"kl": 16.841474533081055,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -35149621.996965095,
|
||
|
|
"logits/rejected": -36465748.096618354,
|
||
|
|
"logps/chosen": -483.0319613050076,
|
||
|
|
"logps/rejected": -372.63259863123994,
|
||
|
|
"loss": 0.6088,
|
||
|
|
"loss/base_kto": 3.2649993896484375,
|
||
|
|
"metrics/advantage_var": 618.9137573242188,
|
||
|
|
"regularization/lipschitz_norm": 1634.1199951171875,
|
||
|
|
"regularization/mmd": 0.23276524245738983,
|
||
|
|
"regularization/rkhs_norm": 277.1014709472656,
|
||
|
|
"regularization/w1": 1.3726608753204346,
|
||
|
|
"rewards/chosen": 0.5679234228293342,
|
||
|
|
"rewards/margins": 0.8084328491735622,
|
||
|
|
"rewards/rejected": -0.24050942634422806,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 17.503400802612305,
|
||
|
|
"kl": 12.907358169555664,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35886561.196747966,
|
||
|
|
"logits/rejected": -37689809.03458647,
|
||
|
|
"logps/chosen": -504.8221544715447,
|
||
|
|
"logps/rejected": -370.4760808270677,
|
||
|
|
"loss": 0.6046,
|
||
|
|
"loss/base_kto": 3.331488847732544,
|
||
|
|
"metrics/advantage_var": 532.1181640625,
|
||
|
|
"regularization/lipschitz_norm": 1535.2772216796875,
|
||
|
|
"regularization/mmd": 0.21579065918922424,
|
||
|
|
"regularization/rkhs_norm": 256.8936462402344,
|
||
|
|
"regularization/w1": 1.28963303565979,
|
||
|
|
"rewards/chosen": 0.3988066882621951,
|
||
|
|
"rewards/margins": 0.7130544047213117,
|
||
|
|
"rewards/rejected": -0.31424771645911653,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 15.293237686157227,
|
||
|
|
"kl": 11.60252857208252,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -35524564.091883615,
|
||
|
|
"logits/rejected": -36040707.266347684,
|
||
|
|
"logps/chosen": -466.69094563552835,
|
||
|
|
"logps/rejected": -362.69913775917064,
|
||
|
|
"loss": 0.5981,
|
||
|
|
"loss/base_kto": 3.279315233230591,
|
||
|
|
"metrics/advantage_var": 587.1659545898438,
|
||
|
|
"regularization/lipschitz_norm": 1533.8731689453125,
|
||
|
|
"regularization/mmd": 0.2168503999710083,
|
||
|
|
"regularization/rkhs_norm": 258.1552429199219,
|
||
|
|
"regularization/w1": 1.288453459739685,
|
||
|
|
"rewards/chosen": 0.4705057187978561,
|
||
|
|
"rewards/margins": 0.7924285199357772,
|
||
|
|
"rewards/rejected": -0.32192280113792116,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 18.3332462310791,
|
||
|
|
"kl": 16.38279151916504,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -34956592.68711656,
|
||
|
|
"logits/rejected": -35879792.50955414,
|
||
|
|
"logps/chosen": -495.4688458588957,
|
||
|
|
"logps/rejected": -384.3523835589172,
|
||
|
|
"loss": 0.6051,
|
||
|
|
"loss/base_kto": 3.2959625720977783,
|
||
|
|
"metrics/advantage_var": 565.0421752929688,
|
||
|
|
"regularization/lipschitz_norm": 1574.880615234375,
|
||
|
|
"regularization/mmd": 0.2218790054321289,
|
||
|
|
"regularization/rkhs_norm": 264.14166259765625,
|
||
|
|
"regularization/w1": 1.322899580001831,
|
||
|
|
"rewards/chosen": 0.5170884395669575,
|
||
|
|
"rewards/margins": 0.7516949941207244,
|
||
|
|
"rewards/rejected": -0.23460655455376692,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 17.840185165405273,
|
||
|
|
"kl": 12.295026779174805,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34747737.620094195,
|
||
|
|
"logits/rejected": -36313377.841368586,
|
||
|
|
"logps/chosen": -491.57520604395603,
|
||
|
|
"logps/rejected": -392.43475408242614,
|
||
|
|
"loss": 0.6145,
|
||
|
|
"loss/base_kto": 3.304518222808838,
|
||
|
|
"metrics/advantage_var": 638.2750244140625,
|
||
|
|
"regularization/lipschitz_norm": 1647.8433837890625,
|
||
|
|
"regularization/mmd": 0.22736597061157227,
|
||
|
|
"regularization/rkhs_norm": 270.6737976074219,
|
||
|
|
"regularization/w1": 1.3841885328292847,
|
||
|
|
"rewards/chosen": 0.42322218099980374,
|
||
|
|
"rewards/margins": 0.7401256586937797,
|
||
|
|
"rewards/rejected": -0.316903477693976,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 17.464134216308594,
|
||
|
|
"kl": 14.800549507141113,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -36132098.84896661,
|
||
|
|
"logits/rejected": -36597973.92319509,
|
||
|
|
"logps/chosen": -502.48802662957075,
|
||
|
|
"logps/rejected": -370.64122503840247,
|
||
|
|
"loss": 0.6016,
|
||
|
|
"loss/base_kto": 3.296837568283081,
|
||
|
|
"metrics/advantage_var": 585.802490234375,
|
||
|
|
"regularization/lipschitz_norm": 1539.325439453125,
|
||
|
|
"regularization/mmd": 0.22298672795295715,
|
||
|
|
"regularization/rkhs_norm": 265.46038818359375,
|
||
|
|
"regularization/w1": 1.2930333614349365,
|
||
|
|
"rewards/chosen": 0.5013735821212987,
|
||
|
|
"rewards/margins": 0.7788224325977291,
|
||
|
|
"rewards/rejected": -0.2774488504764305,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 20.29445457458496,
|
||
|
|
"kl": 12.784585952758789,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -34617143.72239748,
|
||
|
|
"logits/rejected": -36968910.86068112,
|
||
|
|
"logps/chosen": -474.6808458201893,
|
||
|
|
"logps/rejected": -385.672310371517,
|
||
|
|
"loss": 0.6063,
|
||
|
|
"loss/base_kto": 3.296652317047119,
|
||
|
|
"metrics/advantage_var": 653.3554077148438,
|
||
|
|
"regularization/lipschitz_norm": 1586.092041015625,
|
||
|
|
"regularization/mmd": 0.22120271623134613,
|
||
|
|
"regularization/rkhs_norm": 263.3365478515625,
|
||
|
|
"regularization/w1": 1.3323172330856323,
|
||
|
|
"rewards/chosen": 0.3795992493253401,
|
||
|
|
"rewards/margins": 0.7589946072899179,
|
||
|
|
"rewards/rejected": -0.3793953579645777,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 22.748767852783203,
|
||
|
|
"kl": 13.172281265258789,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -34598445.65605096,
|
||
|
|
"logits/rejected": -36255740.858895704,
|
||
|
|
"logps/chosen": -472.8190187101911,
|
||
|
|
"logps/rejected": -365.1085601993865,
|
||
|
|
"loss": 0.6211,
|
||
|
|
"loss/base_kto": 3.232517719268799,
|
||
|
|
"metrics/advantage_var": 774.0167846679688,
|
||
|
|
"regularization/lipschitz_norm": 1770.9593505859375,
|
||
|
|
"regularization/mmd": 0.24844610691070557,
|
||
|
|
"regularization/rkhs_norm": 295.7691650390625,
|
||
|
|
"regularization/w1": 1.4876058101654053,
|
||
|
|
"rewards/chosen": 0.47049406987086984,
|
||
|
|
"rewards/margins": 0.887582631133691,
|
||
|
|
"rewards/rejected": -0.4170885612628211,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 16.23194694519043,
|
||
|
|
"kl": 11.853745460510254,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34278380.04197901,
|
||
|
|
"logits/rejected": -36623218.009787925,
|
||
|
|
"logps/chosen": -459.3505434782609,
|
||
|
|
"logps/rejected": -375.00741741435564,
|
||
|
|
"loss": 0.5884,
|
||
|
|
"loss/base_kto": 3.3125159740448,
|
||
|
|
"metrics/advantage_var": 457.6903381347656,
|
||
|
|
"regularization/lipschitz_norm": 1415.830078125,
|
||
|
|
"regularization/mmd": 0.2050737589597702,
|
||
|
|
"regularization/rkhs_norm": 244.1354522705078,
|
||
|
|
"regularization/w1": 1.1892971992492676,
|
||
|
|
"rewards/chosen": 0.46317237499414354,
|
||
|
|
"rewards/margins": 0.7178690949867517,
|
||
|
|
"rewards/rejected": -0.25469671999260807,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 15.1323823928833,
|
||
|
|
"kl": 23.489944458007812,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34825587.93199382,
|
||
|
|
"logits/rejected": -35962562.93206951,
|
||
|
|
"logps/chosen": -500.9868624420402,
|
||
|
|
"logps/rejected": -393.5623518957346,
|
||
|
|
"loss": 0.6162,
|
||
|
|
"loss/base_kto": 3.2950799465179443,
|
||
|
|
"metrics/advantage_var": 616.2437744140625,
|
||
|
|
"regularization/lipschitz_norm": 1670.2127685546875,
|
||
|
|
"regularization/mmd": 0.23115836083889008,
|
||
|
|
"regularization/rkhs_norm": 275.1884765625,
|
||
|
|
"regularization/w1": 1.402978777885437,
|
||
|
|
"rewards/chosen": 0.6797178760746716,
|
||
|
|
"rewards/margins": 0.745549083060796,
|
||
|
|
"rewards/rejected": -0.06583120698612448,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 19.21268081665039,
|
||
|
|
"kl": 14.270586967468262,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -35360262.4,
|
||
|
|
"logits/rejected": -34184057.6,
|
||
|
|
"logps/chosen": -508.236328125,
|
||
|
|
"logps/rejected": -381.9216064453125,
|
||
|
|
"loss": 0.6007,
|
||
|
|
"loss/base_kto": 3.2953529357910156,
|
||
|
|
"metrics/advantage_var": 574.7315673828125,
|
||
|
|
"regularization/lipschitz_norm": 1533.970947265625,
|
||
|
|
"regularization/mmd": 0.22182214260101318,
|
||
|
|
"regularization/rkhs_norm": 264.073974609375,
|
||
|
|
"regularization/w1": 1.288535475730896,
|
||
|
|
"rewards/chosen": 0.5230182647705078,
|
||
|
|
"rewards/margins": 0.7322259902954102,
|
||
|
|
"rewards/rejected": -0.20920772552490235,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 21.343929290771484,
|
||
|
|
"kl": 10.648024559020996,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -35289440.55086072,
|
||
|
|
"logits/rejected": -36280978.17160686,
|
||
|
|
"logps/chosen": -467.39578442879497,
|
||
|
|
"logps/rejected": -375.72603841653665,
|
||
|
|
"loss": 0.6074,
|
||
|
|
"loss/base_kto": 3.3544445037841797,
|
||
|
|
"metrics/advantage_var": 541.89599609375,
|
||
|
|
"regularization/lipschitz_norm": 1533.8331298828125,
|
||
|
|
"regularization/mmd": 0.21611042320728302,
|
||
|
|
"regularization/rkhs_norm": 257.2743225097656,
|
||
|
|
"regularization/w1": 1.2884197235107422,
|
||
|
|
"rewards/chosen": 0.35909945826761785,
|
||
|
|
"rewards/margins": 0.7024860800850297,
|
||
|
|
"rewards/rejected": -0.34338662181741175,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 18.35866355895996,
|
||
|
|
"kl": 9.532137870788574,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -35452342.85714286,
|
||
|
|
"logits/rejected": -36286360.95597484,
|
||
|
|
"logps/chosen": -464.1807065217391,
|
||
|
|
"logps/rejected": -384.552353577044,
|
||
|
|
"loss": 0.6032,
|
||
|
|
"loss/base_kto": 3.2651467323303223,
|
||
|
|
"metrics/advantage_var": 594.4240112304688,
|
||
|
|
"regularization/lipschitz_norm": 1588.4176025390625,
|
||
|
|
"regularization/mmd": 0.22581540048122406,
|
||
|
|
"regularization/rkhs_norm": 268.827880859375,
|
||
|
|
"regularization/w1": 1.334270715713501,
|
||
|
|
"rewards/chosen": 0.40784947176157316,
|
||
|
|
"rewards/margins": 0.8419069354015103,
|
||
|
|
"rewards/rejected": -0.4340574636399371,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 18.201684951782227,
|
||
|
|
"kl": 13.207061767578125,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -35922484.08137715,
|
||
|
|
"logits/rejected": -37026456.56162246,
|
||
|
|
"logps/chosen": -491.82042253521126,
|
||
|
|
"logps/rejected": -376.1210267160686,
|
||
|
|
"loss": 0.6179,
|
||
|
|
"loss/base_kto": 3.338420867919922,
|
||
|
|
"metrics/advantage_var": 639.4461059570312,
|
||
|
|
"regularization/lipschitz_norm": 1636.497314453125,
|
||
|
|
"regularization/mmd": 0.2301218956708908,
|
||
|
|
"regularization/rkhs_norm": 273.95465087890625,
|
||
|
|
"regularization/w1": 1.3746576309204102,
|
||
|
|
"rewards/chosen": 0.4131676520167009,
|
||
|
|
"rewards/margins": 0.6693103501172031,
|
||
|
|
"rewards/rejected": -0.25614269810050216,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 19.10555648803711,
|
||
|
|
"kl": 9.879256248474121,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -36302703.50920245,
|
||
|
|
"logits/rejected": -36876839.133757964,
|
||
|
|
"logps/chosen": -489.24640529141107,
|
||
|
|
"logps/rejected": -391.04757165605093,
|
||
|
|
"loss": 0.6211,
|
||
|
|
"loss/base_kto": 3.3606956005096436,
|
||
|
|
"metrics/advantage_var": 647.4120483398438,
|
||
|
|
"regularization/lipschitz_norm": 1641.331298828125,
|
||
|
|
"regularization/mmd": 0.22900962829589844,
|
||
|
|
"regularization/rkhs_norm": 272.6305236816406,
|
||
|
|
"regularization/w1": 1.378718376159668,
|
||
|
|
"rewards/chosen": 0.29833345471715633,
|
||
|
|
"rewards/margins": 0.7447196097666686,
|
||
|
|
"rewards/rejected": -0.44638615504951235,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 14.509907722473145,
|
||
|
|
"kl": 14.842035293579102,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -34713587.29922481,
|
||
|
|
"logits/rejected": -36448175.37007874,
|
||
|
|
"logps/chosen": -477.756007751938,
|
||
|
|
"logps/rejected": -369.9964566929134,
|
||
|
|
"loss": 0.5982,
|
||
|
|
"loss/base_kto": 3.300563097000122,
|
||
|
|
"metrics/advantage_var": 547.6964721679688,
|
||
|
|
"regularization/lipschitz_norm": 1511.495849609375,
|
||
|
|
"regularization/mmd": 0.21576833724975586,
|
||
|
|
"regularization/rkhs_norm": 256.8670959472656,
|
||
|
|
"regularization/w1": 1.2696565389633179,
|
||
|
|
"rewards/chosen": 0.542132804929748,
|
||
|
|
"rewards/margins": 0.7534206563384587,
|
||
|
|
"rewards/rejected": -0.21128785140871062,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 20.434032440185547,
|
||
|
|
"kl": 10.717257499694824,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -35180438.974358976,
|
||
|
|
"logits/rejected": -36399558.74230146,
|
||
|
|
"logps/chosen": -490.4654977375566,
|
||
|
|
"logps/rejected": -363.00486223662887,
|
||
|
|
"loss": 0.611,
|
||
|
|
"loss/base_kto": 3.3132057189941406,
|
||
|
|
"metrics/advantage_var": 643.6455688476562,
|
||
|
|
"regularization/lipschitz_norm": 1605.9615478515625,
|
||
|
|
"regularization/mmd": 0.225395068526268,
|
||
|
|
"regularization/rkhs_norm": 268.3274841308594,
|
||
|
|
"regularization/w1": 1.3490076065063477,
|
||
|
|
"rewards/chosen": 0.47210012122336914,
|
||
|
|
"rewards/margins": 0.7773512566553358,
|
||
|
|
"rewards/rejected": -0.3052511354319667,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 19.002588272094727,
|
||
|
|
"kl": 16.595155715942383,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -34794416.98267716,
|
||
|
|
"logits/rejected": -35285217.43875969,
|
||
|
|
"logps/chosen": -465.85767716535435,
|
||
|
|
"logps/rejected": -377.64580910852715,
|
||
|
|
"loss": 0.5981,
|
||
|
|
"loss/base_kto": 3.2985668182373047,
|
||
|
|
"metrics/advantage_var": 573.4653930664062,
|
||
|
|
"regularization/lipschitz_norm": 1511.4486083984375,
|
||
|
|
"regularization/mmd": 0.21655896306037903,
|
||
|
|
"regularization/rkhs_norm": 257.8083190917969,
|
||
|
|
"regularization/w1": 1.269616961479187,
|
||
|
|
"rewards/chosen": 0.5256030946265994,
|
||
|
|
"rewards/margins": 0.7473348039516357,
|
||
|
|
"rewards/rejected": -0.22173170932503633,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 24.03455352783203,
|
||
|
|
"kl": 15.612261772155762,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -35529845.71968504,
|
||
|
|
"logits/rejected": -37056543.751937985,
|
||
|
|
"logps/chosen": -464.00305118110236,
|
||
|
|
"logps/rejected": -385.03032945736436,
|
||
|
|
"loss": 0.6003,
|
||
|
|
"loss/base_kto": 3.2823617458343506,
|
||
|
|
"metrics/advantage_var": 530.997802734375,
|
||
|
|
"regularization/lipschitz_norm": 1552.0469970703125,
|
||
|
|
"regularization/mmd": 0.21617642045021057,
|
||
|
|
"regularization/rkhs_norm": 257.3528747558594,
|
||
|
|
"regularization/w1": 1.303719401359558,
|
||
|
|
"rewards/chosen": 0.4894244336706447,
|
||
|
|
"rewards/margins": 0.7310083953348889,
|
||
|
|
"rewards/rejected": -0.24158396166424417,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 17.24437141418457,
|
||
|
|
"kl": 9.225619316101074,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -34000927.950078,
|
||
|
|
"logits/rejected": -36725998.77308294,
|
||
|
|
"logps/chosen": -487.3340483619345,
|
||
|
|
"logps/rejected": -378.5811081768388,
|
||
|
|
"loss": 0.5965,
|
||
|
|
"loss/base_kto": 3.292456865310669,
|
||
|
|
"metrics/advantage_var": 584.4599609375,
|
||
|
|
"regularization/lipschitz_norm": 1504.295166015625,
|
||
|
|
"regularization/mmd": 0.21610970795154572,
|
||
|
|
"regularization/rkhs_norm": 257.27349853515625,
|
||
|
|
"regularization/w1": 1.2636079788208008,
|
||
|
|
"rewards/chosen": 0.38171298641496443,
|
||
|
|
"rewards/margins": 0.7733153130483761,
|
||
|
|
"rewards/rejected": -0.3916023266334116,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 19.111923217773438,
|
||
|
|
"kl": 10.950237274169922,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -34312166.832565285,
|
||
|
|
"logits/rejected": -36734572.26073132,
|
||
|
|
"logps/chosen": -501.1262960829493,
|
||
|
|
"logps/rejected": -350.72848767885534,
|
||
|
|
"loss": 0.6082,
|
||
|
|
"loss/base_kto": 3.311518907546997,
|
||
|
|
"metrics/advantage_var": 581.57373046875,
|
||
|
|
"regularization/lipschitz_norm": 1586.0428466796875,
|
||
|
|
"regularization/mmd": 0.22160875797271729,
|
||
|
|
"regularization/rkhs_norm": 263.8199768066406,
|
||
|
|
"regularization/w1": 1.3322757482528687,
|
||
|
|
"rewards/chosen": 0.4473965083765361,
|
||
|
|
"rewards/margins": 0.7809996804968411,
|
||
|
|
"rewards/rejected": -0.33360317212030505,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 21.935710906982422,
|
||
|
|
"kl": 12.454663276672363,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -35260641.43875969,
|
||
|
|
"logits/rejected": -36672987.716535434,
|
||
|
|
"logps/chosen": -496.70915697674417,
|
||
|
|
"logps/rejected": -395.18590059055117,
|
||
|
|
"loss": 0.6079,
|
||
|
|
"loss/base_kto": 3.2811696529388428,
|
||
|
|
"metrics/advantage_var": 641.7221069335938,
|
||
|
|
"regularization/lipschitz_norm": 1609.4114990234375,
|
||
|
|
"regularization/mmd": 0.2302233725786209,
|
||
|
|
"regularization/rkhs_norm": 274.075439453125,
|
||
|
|
"regularization/w1": 1.3519057035446167,
|
||
|
|
"rewards/chosen": 0.4949009621790213,
|
||
|
|
"rewards/margins": 0.7782938248143559,
|
||
|
|
"rewards/rejected": -0.28339286263533464,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 15.439901351928711,
|
||
|
|
"kl": 7.926913738250732,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -33904806.66040689,
|
||
|
|
"logits/rejected": -35364937.48517941,
|
||
|
|
"logps/chosen": -483.5994718309859,
|
||
|
|
"logps/rejected": -365.2688182527301,
|
||
|
|
"loss": 0.6002,
|
||
|
|
"loss/base_kto": 3.3556129932403564,
|
||
|
|
"metrics/advantage_var": 503.3187561035156,
|
||
|
|
"regularization/lipschitz_norm": 1472.1734619140625,
|
||
|
|
"regularization/mmd": 0.2091762274503708,
|
||
|
|
"regularization/rkhs_norm": 249.01931762695312,
|
||
|
|
"regularization/w1": 1.2366255521774292,
|
||
|
|
"rewards/chosen": 0.2995943120201242,
|
||
|
|
"rewards/margins": 0.6907918617775832,
|
||
|
|
"rewards/rejected": -0.3911975497574591,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 19.90335464477539,
|
||
|
|
"kl": 11.342299461364746,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -34794194.271755725,
|
||
|
|
"logits/rejected": -37152240.4352,
|
||
|
|
"logps/chosen": -504.8614503816794,
|
||
|
|
"logps/rejected": -385.39135,
|
||
|
|
"loss": 0.6081,
|
||
|
|
"loss/base_kto": 3.30735182762146,
|
||
|
|
"metrics/advantage_var": 541.2826538085938,
|
||
|
|
"regularization/lipschitz_norm": 1590.8343505859375,
|
||
|
|
"regularization/mmd": 0.22089329361915588,
|
||
|
|
"regularization/rkhs_norm": 262.9682312011719,
|
||
|
|
"regularization/w1": 1.3363008499145508,
|
||
|
|
"rewards/chosen": 0.45415178837666986,
|
||
|
|
"rewards/margins": 0.7193135803688573,
|
||
|
|
"rewards/rejected": -0.2651617919921875,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 16.884435653686523,
|
||
|
|
"kl": 11.711886405944824,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -36216232.903020665,
|
||
|
|
"logits/rejected": -37212546.94930875,
|
||
|
|
"logps/chosen": -472.83575119236883,
|
||
|
|
"logps/rejected": -383.0213853686636,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.303622007369995,
|
||
|
|
"metrics/advantage_var": 564.1619262695312,
|
||
|
|
"regularization/lipschitz_norm": 1524.880859375,
|
||
|
|
"regularization/mmd": 0.218998983502388,
|
||
|
|
"regularization/rkhs_norm": 260.7131042480469,
|
||
|
|
"regularization/w1": 1.2808998823165894,
|
||
|
|
"rewards/chosen": 0.47668621991131754,
|
||
|
|
"rewards/margins": 0.7623285010173083,
|
||
|
|
"rewards/rejected": -0.28564228110599077,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 18.357881546020508,
|
||
|
|
"kl": 10.192360877990723,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -36429468.24413145,
|
||
|
|
"logits/rejected": -36543176.48673947,
|
||
|
|
"logps/chosen": -478.88468309859155,
|
||
|
|
"logps/rejected": -376.95205245709826,
|
||
|
|
"loss": 0.6054,
|
||
|
|
"loss/base_kto": 3.2851758003234863,
|
||
|
|
"metrics/advantage_var": 591.59375,
|
||
|
|
"regularization/lipschitz_norm": 1588.7135009765625,
|
||
|
|
"regularization/mmd": 0.22349607944488525,
|
||
|
|
"regularization/rkhs_norm": 266.0667419433594,
|
||
|
|
"regularization/w1": 1.3345192670822144,
|
||
|
|
"rewards/chosen": 0.3761246849860011,
|
||
|
|
"rewards/margins": 0.7807911987949909,
|
||
|
|
"rewards/rejected": -0.4046665138089899,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 16.401823043823242,
|
||
|
|
"kl": 12.415465354919434,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -35214832.0,
|
||
|
|
"logits/rejected": -36070636.8,
|
||
|
|
"logps/chosen": -514.97548828125,
|
||
|
|
"logps/rejected": -359.206982421875,
|
||
|
|
"loss": 0.5968,
|
||
|
|
"loss/base_kto": 3.344966173171997,
|
||
|
|
"metrics/advantage_var": 478.7113342285156,
|
||
|
|
"regularization/lipschitz_norm": 1459.605224609375,
|
||
|
|
"regularization/mmd": 0.20345313847064972,
|
||
|
|
"regularization/rkhs_norm": 242.2061004638672,
|
||
|
|
"regularization/w1": 1.226068377494812,
|
||
|
|
"rewards/chosen": 0.43894271850585936,
|
||
|
|
"rewards/margins": 0.7065156459808349,
|
||
|
|
"rewards/rejected": -0.2675729274749756,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 14.401206016540527,
|
||
|
|
"kl": 8.821708679199219,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34049681.69105691,
|
||
|
|
"logits/rejected": -35270160.16842105,
|
||
|
|
"logps/chosen": -447.6968495934959,
|
||
|
|
"logps/rejected": -391.48869830827067,
|
||
|
|
"loss": 0.5963,
|
||
|
|
"loss/base_kto": 3.3092942237854004,
|
||
|
|
"metrics/advantage_var": 565.7273559570312,
|
||
|
|
"regularization/lipschitz_norm": 1480.9267578125,
|
||
|
|
"regularization/mmd": 0.2175239622592926,
|
||
|
|
"regularization/rkhs_norm": 258.95709228515625,
|
||
|
|
"regularization/w1": 1.2439783811569214,
|
||
|
|
"rewards/chosen": 0.27671960349974595,
|
||
|
|
"rewards/margins": 0.7484900635855072,
|
||
|
|
"rewards/rejected": -0.4717704600857613,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 15.626259803771973,
|
||
|
|
"kl": 7.557384490966797,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -35209195.31313131,
|
||
|
|
"logits/rejected": -35694138.21574344,
|
||
|
|
"logps/chosen": -475.2208543771044,
|
||
|
|
"logps/rejected": -369.07438957725947,
|
||
|
|
"loss": 0.6022,
|
||
|
|
"loss/base_kto": 3.277252674102783,
|
||
|
|
"metrics/advantage_var": 589.4804077148438,
|
||
|
|
"regularization/lipschitz_norm": 1572.3720703125,
|
||
|
|
"regularization/mmd": 0.2191886156797409,
|
||
|
|
"regularization/rkhs_norm": 260.9388122558594,
|
||
|
|
"regularization/w1": 1.320792555809021,
|
||
|
|
"rewards/chosen": 0.22359299579453387,
|
||
|
|
"rewards/margins": 0.796166821641837,
|
||
|
|
"rewards/rejected": -0.5725738258473032,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 12.732769012451172,
|
||
|
|
"kl": 8.560650825500488,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -34714454.62822458,
|
||
|
|
"logits/rejected": -34373196.6763285,
|
||
|
|
"logps/chosen": -453.97486722306525,
|
||
|
|
"logps/rejected": -362.400462962963,
|
||
|
|
"loss": 0.6011,
|
||
|
|
"loss/base_kto": 3.4078118801116943,
|
||
|
|
"metrics/advantage_var": 482.5973205566406,
|
||
|
|
"regularization/lipschitz_norm": 1423.417236328125,
|
||
|
|
"regularization/mmd": 0.204934224486351,
|
||
|
|
"regularization/rkhs_norm": 243.9692840576172,
|
||
|
|
"regularization/w1": 1.195670485496521,
|
||
|
|
"rewards/chosen": 0.26198312762293724,
|
||
|
|
"rewards/margins": 0.6308549915363582,
|
||
|
|
"rewards/rejected": -0.3688718639134209,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 17.801597595214844,
|
||
|
|
"kl": 11.835659980773926,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -34583271.27725857,
|
||
|
|
"logits/rejected": -35379042.70846395,
|
||
|
|
"logps/chosen": -466.01343457943926,
|
||
|
|
"logps/rejected": -364.40985011755487,
|
||
|
|
"loss": 0.6076,
|
||
|
|
"loss/base_kto": 3.3887290954589844,
|
||
|
|
"metrics/advantage_var": 506.2080993652344,
|
||
|
|
"regularization/lipschitz_norm": 1503.4544677734375,
|
||
|
|
"regularization/mmd": 0.209140807390213,
|
||
|
|
"regularization/rkhs_norm": 248.97715759277344,
|
||
|
|
"regularization/w1": 1.262901782989502,
|
||
|
|
"rewards/chosen": 0.3550300954658294,
|
||
|
|
"rewards/margins": 0.6660641993855492,
|
||
|
|
"rewards/rejected": -0.3110341039197198,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 19.292692184448242,
|
||
|
|
"kl": 11.200427055358887,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -35509931.228995055,
|
||
|
|
"logits/rejected": -36423110.94205052,
|
||
|
|
"logps/chosen": -501.6638179571664,
|
||
|
|
"logps/rejected": -374.9111952080238,
|
||
|
|
"loss": 0.603,
|
||
|
|
"loss/base_kto": 3.302513837814331,
|
||
|
|
"metrics/advantage_var": 571.0662231445312,
|
||
|
|
"regularization/lipschitz_norm": 1553.4810791015625,
|
||
|
|
"regularization/mmd": 0.21647481620311737,
|
||
|
|
"regularization/rkhs_norm": 257.7080993652344,
|
||
|
|
"regularization/w1": 1.3049242496490479,
|
||
|
|
"rewards/chosen": 0.4032447978338653,
|
||
|
|
"rewards/margins": 0.7491951923041709,
|
||
|
|
"rewards/rejected": -0.3459503944703055,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 16.644149780273438,
|
||
|
|
"kl": 9.445067405700684,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34352410.256410256,
|
||
|
|
"logits/rejected": -36357946.71559633,
|
||
|
|
"logps/chosen": -493.4545272435897,
|
||
|
|
"logps/rejected": -375.8651567278287,
|
||
|
|
"loss": 0.5747,
|
||
|
|
"loss/base_kto": 3.1822478771209717,
|
||
|
|
"metrics/advantage_var": 513.93603515625,
|
||
|
|
"regularization/lipschitz_norm": 1433.6219482421875,
|
||
|
|
"regularization/mmd": 0.21107494831085205,
|
||
|
|
"regularization/rkhs_norm": 251.2797088623047,
|
||
|
|
"regularization/w1": 1.204242467880249,
|
||
|
|
"rewards/chosen": 0.44784858899238783,
|
||
|
|
"rewards/margins": 0.8653953191505491,
|
||
|
|
"rewards/rejected": -0.4175467301581613,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 13.901891708374023,
|
||
|
|
"kl": 7.060243129730225,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34100639.65163297,
|
||
|
|
"logits/rejected": -35459702.15384615,
|
||
|
|
"logps/chosen": -475.0867029548989,
|
||
|
|
"logps/rejected": -364.4067896389325,
|
||
|
|
"loss": 0.5598,
|
||
|
|
"loss/base_kto": 3.289045810699463,
|
||
|
|
"metrics/advantage_var": 371.1310729980469,
|
||
|
|
"regularization/lipschitz_norm": 1202.791748046875,
|
||
|
|
"regularization/mmd": 0.1786099672317505,
|
||
|
|
"regularization/rkhs_norm": 212.6309051513672,
|
||
|
|
"regularization/w1": 1.0103449821472168,
|
||
|
|
"rewards/chosen": 0.30485949953829705,
|
||
|
|
"rewards/margins": 0.7512412645715338,
|
||
|
|
"rewards/rejected": -0.44638176503323684,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 16.676607131958008,
|
||
|
|
"kl": 10.98784351348877,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -36444727.39672131,
|
||
|
|
"logits/rejected": -36727670.44776119,
|
||
|
|
"logps/chosen": -487.9197745901639,
|
||
|
|
"logps/rejected": -376.894239738806,
|
||
|
|
"loss": 0.5601,
|
||
|
|
"loss/base_kto": 3.2293717861175537,
|
||
|
|
"metrics/advantage_var": 413.7928771972656,
|
||
|
|
"regularization/lipschitz_norm": 1264.6346435546875,
|
||
|
|
"regularization/mmd": 0.18891631066799164,
|
||
|
|
"regularization/rkhs_norm": 224.9003448486328,
|
||
|
|
"regularization/w1": 1.0622931718826294,
|
||
|
|
"rewards/chosen": 0.48784089635630123,
|
||
|
|
"rewards/margins": 0.8099219583315811,
|
||
|
|
"rewards/rejected": -0.32208106197527986,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 17.610885620117188,
|
||
|
|
"kl": 6.0838446617126465,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -35514506.63384616,
|
||
|
|
"logits/rejected": -35825153.625396825,
|
||
|
|
"logps/chosen": -467.02221153846153,
|
||
|
|
"logps/rejected": -377.14201388888887,
|
||
|
|
"loss": 0.561,
|
||
|
|
"loss/base_kto": 3.218971014022827,
|
||
|
|
"metrics/advantage_var": 421.5039978027344,
|
||
|
|
"regularization/lipschitz_norm": 1282.7529296875,
|
||
|
|
"regularization/mmd": 0.19190359115600586,
|
||
|
|
"regularization/rkhs_norm": 228.45664978027344,
|
||
|
|
"regularization/w1": 1.077512502670288,
|
||
|
|
"rewards/chosen": 0.33190122164212743,
|
||
|
|
"rewards/margins": 0.8490128636622167,
|
||
|
|
"rewards/rejected": -0.5171116420200893,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 17.137195587158203,
|
||
|
|
"kl": 11.394990921020508,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -32733561.263157893,
|
||
|
|
"logits/rejected": -33761583.646687694,
|
||
|
|
"logps/chosen": -480.812258126935,
|
||
|
|
"logps/rejected": -358.0452484227129,
|
||
|
|
"loss": 0.5583,
|
||
|
|
"loss/base_kto": 3.1753804683685303,
|
||
|
|
"metrics/advantage_var": 420.86236572265625,
|
||
|
|
"regularization/lipschitz_norm": 1304.782470703125,
|
||
|
|
"regularization/mmd": 0.1950908750295639,
|
||
|
|
"regularization/rkhs_norm": 232.25100708007812,
|
||
|
|
"regularization/w1": 1.0960171222686768,
|
||
|
|
"rewards/chosen": 0.5098609570010159,
|
||
|
|
"rewards/margins": 0.8511433412537145,
|
||
|
|
"rewards/rejected": -0.34128238425269863,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 19.45854949951172,
|
||
|
|
"kl": 11.109087944030762,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -35332594.24960505,
|
||
|
|
"logits/rejected": -37018087.4683153,
|
||
|
|
"logps/chosen": -473.1962875197472,
|
||
|
|
"logps/rejected": -361.85951989953634,
|
||
|
|
"loss": 0.5621,
|
||
|
|
"loss/base_kto": 3.174731969833374,
|
||
|
|
"metrics/advantage_var": 445.5321350097656,
|
||
|
|
"regularization/lipschitz_norm": 1338.1480712890625,
|
||
|
|
"regularization/mmd": 0.198368638753891,
|
||
|
|
"regularization/rkhs_norm": 236.1531219482422,
|
||
|
|
"regularization/w1": 1.1240442991256714,
|
||
|
|
"rewards/chosen": 0.5304546145277942,
|
||
|
|
"rewards/margins": 0.8823722174441992,
|
||
|
|
"rewards/rejected": -0.35191760291640506,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 15.302573204040527,
|
||
|
|
"kl": 9.516722679138184,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -33546011.069486406,
|
||
|
|
"logits/rejected": -34918877.20388349,
|
||
|
|
"logps/chosen": -449.7597243202417,
|
||
|
|
"logps/rejected": -382.87616302589,
|
||
|
|
"loss": 0.5555,
|
||
|
|
"loss/base_kto": 3.19384765625,
|
||
|
|
"metrics/advantage_var": 388.11187744140625,
|
||
|
|
"regularization/lipschitz_norm": 1269.3221435546875,
|
||
|
|
"regularization/mmd": 0.18409369885921478,
|
||
|
|
"regularization/rkhs_norm": 219.15916442871094,
|
||
|
|
"regularization/w1": 1.0662306547164917,
|
||
|
|
"rewards/chosen": 0.4260016035097243,
|
||
|
|
"rewards/margins": 0.8581687308041721,
|
||
|
|
"rewards/rejected": -0.43216712729444784,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 19.544517517089844,
|
||
|
|
"kl": 11.847509384155273,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34576406.02150538,
|
||
|
|
"logits/rejected": -36275406.753577106,
|
||
|
|
"logps/chosen": -483.99476766513055,
|
||
|
|
"logps/rejected": -385.47076212241655,
|
||
|
|
"loss": 0.5549,
|
||
|
|
"loss/base_kto": 3.1940832138061523,
|
||
|
|
"metrics/advantage_var": 405.37420654296875,
|
||
|
|
"regularization/lipschitz_norm": 1256.4134521484375,
|
||
|
|
"regularization/mmd": 0.18994541466236115,
|
||
|
|
"regularization/rkhs_norm": 226.12548828125,
|
||
|
|
"regularization/w1": 1.0553873777389526,
|
||
|
|
"rewards/chosen": 0.5255388532366071,
|
||
|
|
"rewards/margins": 0.8348792892896179,
|
||
|
|
"rewards/rejected": -0.30934043605301076,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 18.164138793945312,
|
||
|
|
"kl": 13.985907554626465,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -34494572.1344,
|
||
|
|
"logits/rejected": -36497892.641221374,
|
||
|
|
"logps/chosen": -470.2582,
|
||
|
|
"logps/rejected": -379.03122614503815,
|
||
|
|
"loss": 0.5538,
|
||
|
|
"loss/base_kto": 3.1739678382873535,
|
||
|
|
"metrics/advantage_var": 411.597900390625,
|
||
|
|
"regularization/lipschitz_norm": 1267.6138916015625,
|
||
|
|
"regularization/mmd": 0.19165661931037903,
|
||
|
|
"regularization/rkhs_norm": 228.1626434326172,
|
||
|
|
"regularization/w1": 1.064795732498169,
|
||
|
|
"rewards/chosen": 0.5069537109375,
|
||
|
|
"rewards/margins": 0.8373428589799021,
|
||
|
|
"rewards/rejected": -0.3303891480424022,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 20.673307418823242,
|
||
|
|
"kl": 14.906885147094727,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -34981897.46687211,
|
||
|
|
"logits/rejected": -36298154.80190174,
|
||
|
|
"logps/chosen": -508.90716486902926,
|
||
|
|
"logps/rejected": -370.3980536846276,
|
||
|
|
"loss": 0.5631,
|
||
|
|
"loss/base_kto": 3.2059707641601562,
|
||
|
|
"metrics/advantage_var": 383.49420166015625,
|
||
|
|
"regularization/lipschitz_norm": 1323.1077880859375,
|
||
|
|
"regularization/mmd": 0.1872604936361313,
|
||
|
|
"regularization/rkhs_norm": 222.9291534423828,
|
||
|
|
"regularization/w1": 1.1114104986190796,
|
||
|
|
"rewards/chosen": 0.5036289115165158,
|
||
|
|
"rewards/margins": 0.8116960519294887,
|
||
|
|
"rewards/rejected": -0.30806714041297295,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 15.288378715515137,
|
||
|
|
"kl": 12.730751991271973,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -35215665.72070626,
|
||
|
|
"logits/rejected": -36038861.735159814,
|
||
|
|
"logps/chosen": -449.7240168539326,
|
||
|
|
"logps/rejected": -389.68095985540333,
|
||
|
|
"loss": 0.5547,
|
||
|
|
"loss/base_kto": 3.1908655166625977,
|
||
|
|
"metrics/advantage_var": 390.4069519042969,
|
||
|
|
"regularization/lipschitz_norm": 1263.0872802734375,
|
||
|
|
"regularization/mmd": 0.18605269491672516,
|
||
|
|
"regularization/rkhs_norm": 221.49131774902344,
|
||
|
|
"regularization/w1": 1.0609934329986572,
|
||
|
|
"rewards/chosen": 0.4639940583304073,
|
||
|
|
"rewards/margins": 0.8198346861545607,
|
||
|
|
"rewards/rejected": -0.3558406278241533,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 21.061296463012695,
|
||
|
|
"kl": 8.676907539367676,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35120666.52433281,
|
||
|
|
"logits/rejected": -35356200.609642304,
|
||
|
|
"logps/chosen": -478.1992739403454,
|
||
|
|
"logps/rejected": -381.2404986391913,
|
||
|
|
"loss": 0.5609,
|
||
|
|
"loss/base_kto": 3.1834607124328613,
|
||
|
|
"metrics/advantage_var": 495.4214782714844,
|
||
|
|
"regularization/lipschitz_norm": 1316.703369140625,
|
||
|
|
"regularization/mmd": 0.1977226883172989,
|
||
|
|
"regularization/rkhs_norm": 235.38418579101562,
|
||
|
|
"regularization/w1": 1.106030821800232,
|
||
|
|
"rewards/chosen": 0.4273147942319957,
|
||
|
|
"rewards/margins": 0.8885355903810286,
|
||
|
|
"rewards/rejected": -0.46122079614903283,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 17.9831600189209,
|
||
|
|
"kl": 10.530211448669434,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -35472780.43647235,
|
||
|
|
"logits/rejected": -36643013.76104746,
|
||
|
|
"logps/chosen": -481.1157044095665,
|
||
|
|
"logps/rejected": -383.91517491816694,
|
||
|
|
"loss": 0.5617,
|
||
|
|
"loss/base_kto": 3.174682378768921,
|
||
|
|
"metrics/advantage_var": 448.5562438964844,
|
||
|
|
"regularization/lipschitz_norm": 1334.2083740234375,
|
||
|
|
"regularization/mmd": 0.19779978692531586,
|
||
|
|
"regularization/rkhs_norm": 235.47593688964844,
|
||
|
|
"regularization/w1": 1.1207350492477417,
|
||
|
|
"rewards/chosen": 0.5076805764784192,
|
||
|
|
"rewards/margins": 0.8769515879558256,
|
||
|
|
"rewards/rejected": -0.36927101147740643,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 21.398677825927734,
|
||
|
|
"kl": 12.57591438293457,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -34479375.5399361,
|
||
|
|
"logits/rejected": -36327023.16819572,
|
||
|
|
"logps/chosen": -478.4351038338658,
|
||
|
|
"logps/rejected": -344.48184250764524,
|
||
|
|
"loss": 0.5578,
|
||
|
|
"loss/base_kto": 3.173804998397827,
|
||
|
|
"metrics/advantage_var": 423.22979736328125,
|
||
|
|
"regularization/lipschitz_norm": 1305.4921875,
|
||
|
|
"regularization/mmd": 0.19224442541599274,
|
||
|
|
"regularization/rkhs_norm": 228.86241149902344,
|
||
|
|
"regularization/w1": 1.0966134071350098,
|
||
|
|
"rewards/chosen": 0.5006509116663339,
|
||
|
|
"rewards/margins": 0.846831422114358,
|
||
|
|
"rewards/rejected": -0.34618051044802417,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 18.928936004638672,
|
||
|
|
"kl": 9.807095527648926,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -34870442.12140575,
|
||
|
|
"logits/rejected": -36020324.20795107,
|
||
|
|
"logps/chosen": -503.499750399361,
|
||
|
|
"logps/rejected": -350.60540902140673,
|
||
|
|
"loss": 0.5655,
|
||
|
|
"loss/base_kto": 3.243364095687866,
|
||
|
|
"metrics/advantage_var": 417.81378173828125,
|
||
|
|
"regularization/lipschitz_norm": 1292.3668212890625,
|
||
|
|
"regularization/mmd": 0.19472244381904602,
|
||
|
|
"regularization/rkhs_norm": 231.8124237060547,
|
||
|
|
"regularization/w1": 1.0855880975723267,
|
||
|
|
"rewards/chosen": 0.41893561122516476,
|
||
|
|
"rewards/margins": 0.794475688364643,
|
||
|
|
"rewards/rejected": -0.37554007713947823,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 17.10317039489746,
|
||
|
|
"kl": 8.416609764099121,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -33858659.510971785,
|
||
|
|
"logits/rejected": -35395842.39252336,
|
||
|
|
"logps/chosen": -486.0876763322884,
|
||
|
|
"logps/rejected": -372.2388288551402,
|
||
|
|
"loss": 0.5489,
|
||
|
|
"loss/base_kto": 3.211177110671997,
|
||
|
|
"metrics/advantage_var": 358.4283447265625,
|
||
|
|
"regularization/lipschitz_norm": 1190.0291748046875,
|
||
|
|
"regularization/mmd": 0.18022236227989197,
|
||
|
|
"regularization/rkhs_norm": 214.55044555664062,
|
||
|
|
"regularization/w1": 0.9996246695518494,
|
||
|
|
"rewards/chosen": 0.39750503970537815,
|
||
|
|
"rewards/margins": 0.8183580335843696,
|
||
|
|
"rewards/rejected": -0.42085299387899144,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 19.44133758544922,
|
||
|
|
"kl": 8.179915428161621,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -35162959.11251981,
|
||
|
|
"logits/rejected": -36410618.87211094,
|
||
|
|
"logps/chosen": -488.91595681458006,
|
||
|
|
"logps/rejected": -404.3394164098613,
|
||
|
|
"loss": 0.5603,
|
||
|
|
"loss/base_kto": 3.217337131500244,
|
||
|
|
"metrics/advantage_var": 390.8741149902344,
|
||
|
|
"regularization/lipschitz_norm": 1283.2906494140625,
|
||
|
|
"regularization/mmd": 0.18738730251789093,
|
||
|
|
"regularization/rkhs_norm": 223.0801239013672,
|
||
|
|
"regularization/w1": 1.077964186668396,
|
||
|
|
"rewards/chosen": 0.3965188342304502,
|
||
|
|
"rewards/margins": 0.8207049882594852,
|
||
|
|
"rewards/rejected": -0.42418615402903503,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 13.140857696533203,
|
||
|
|
"kl": 12.257731437683105,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -34587002.04173355,
|
||
|
|
"logits/rejected": -35462613.13850837,
|
||
|
|
"logps/chosen": -489.2917837078652,
|
||
|
|
"logps/rejected": -379.15988869863014,
|
||
|
|
"loss": 0.5494,
|
||
|
|
"loss/base_kto": 3.1309659481048584,
|
||
|
|
"metrics/advantage_var": 407.1883239746094,
|
||
|
|
"regularization/lipschitz_norm": 1278.0155029296875,
|
||
|
|
"regularization/mmd": 0.19078731536865234,
|
||
|
|
"regularization/rkhs_norm": 227.12779235839844,
|
||
|
|
"regularization/w1": 1.073533058166504,
|
||
|
|
"rewards/chosen": 0.47789327214273175,
|
||
|
|
"rewards/margins": 0.8930512928125615,
|
||
|
|
"rewards/rejected": -0.4151580206698297,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 17.789430618286133,
|
||
|
|
"kl": 10.181300163269043,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -34432216.64535768,
|
||
|
|
"logits/rejected": -35340343.88443018,
|
||
|
|
"logps/chosen": -475.6851217656012,
|
||
|
|
"logps/rejected": -393.29634831460675,
|
||
|
|
"loss": 0.5581,
|
||
|
|
"loss/base_kto": 3.1766598224639893,
|
||
|
|
"metrics/advantage_var": 422.841796875,
|
||
|
|
"regularization/lipschitz_norm": 1302.3504638671875,
|
||
|
|
"regularization/mmd": 0.19428718090057373,
|
||
|
|
"regularization/rkhs_norm": 231.2942352294922,
|
||
|
|
"regularization/w1": 1.0939743518829346,
|
||
|
|
"rewards/chosen": 0.4563505733031298,
|
||
|
|
"rewards/margins": 0.8831588402260331,
|
||
|
|
"rewards/rejected": -0.4268082669229033,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 17.526042938232422,
|
||
|
|
"kl": 11.04592514038086,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -35837780.23225807,
|
||
|
|
"logits/rejected": -36396584.339393936,
|
||
|
|
"logps/chosen": -485.88896169354837,
|
||
|
|
"logps/rejected": -373.12095170454546,
|
||
|
|
"loss": 0.567,
|
||
|
|
"loss/base_kto": 3.2063674926757812,
|
||
|
|
"metrics/advantage_var": 665.98095703125,
|
||
|
|
"regularization/lipschitz_norm": 1342.6810302734375,
|
||
|
|
"regularization/mmd": 0.20164036750793457,
|
||
|
|
"regularization/rkhs_norm": 240.04808044433594,
|
||
|
|
"regularization/w1": 1.127851963043213,
|
||
|
|
"rewards/chosen": 0.4459624751921623,
|
||
|
|
"rewards/margins": 0.7803076511953584,
|
||
|
|
"rewards/rejected": -0.334345176003196,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 23.38262367248535,
|
||
|
|
"kl": 9.86251449584961,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35313890.86494689,
|
||
|
|
"logits/rejected": -36458852.17391305,
|
||
|
|
"logps/chosen": -493.2988429438543,
|
||
|
|
"logps/rejected": -383.49111815619966,
|
||
|
|
"loss": 0.5563,
|
||
|
|
"loss/base_kto": 3.220690965652466,
|
||
|
|
"metrics/advantage_var": 407.3346862792969,
|
||
|
|
"regularization/lipschitz_norm": 1240.0782470703125,
|
||
|
|
"regularization/mmd": 0.18785487115383148,
|
||
|
|
"regularization/rkhs_norm": 223.6367645263672,
|
||
|
|
"regularization/w1": 1.04166579246521,
|
||
|
|
"rewards/chosen": 0.4422240655230463,
|
||
|
|
"rewards/margins": 0.824145412927686,
|
||
|
|
"rewards/rejected": -0.3819213474046397,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 11.991673469543457,
|
||
|
|
"kl": 11.102667808532715,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -34851144.85214626,
|
||
|
|
"logits/rejected": -36199636.35023042,
|
||
|
|
"logps/chosen": -484.1360294117647,
|
||
|
|
"logps/rejected": -351.56322004608296,
|
||
|
|
"loss": 0.5543,
|
||
|
|
"loss/base_kto": 3.2197282314300537,
|
||
|
|
"metrics/advantage_var": 374.5247497558594,
|
||
|
|
"regularization/lipschitz_norm": 1229.1395263671875,
|
||
|
|
"regularization/mmd": 0.1825108379125595,
|
||
|
|
"regularization/rkhs_norm": 217.27481079101562,
|
||
|
|
"regularization/w1": 1.0324772596359253,
|
||
|
|
"rewards/chosen": 0.462889072436407,
|
||
|
|
"rewards/margins": 0.81789956654553,
|
||
|
|
"rewards/rejected": -0.355010494109123,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 16.491413116455078,
|
||
|
|
"kl": 10.519144058227539,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -34574907.96396396,
|
||
|
|
"logits/rejected": -37384525.5504886,
|
||
|
|
"logps/chosen": -484.92858483483485,
|
||
|
|
"logps/rejected": -366.17457247557,
|
||
|
|
"loss": 0.5599,
|
||
|
|
"loss/base_kto": 3.1814286708831787,
|
||
|
|
"metrics/advantage_var": 463.4703674316406,
|
||
|
|
"regularization/lipschitz_norm": 1307.408203125,
|
||
|
|
"regularization/mmd": 0.19928239285945892,
|
||
|
|
"regularization/rkhs_norm": 237.2409210205078,
|
||
|
|
"regularization/w1": 1.0982228517532349,
|
||
|
|
"rewards/chosen": 0.4894552144918356,
|
||
|
|
"rewards/margins": 0.8710963911939809,
|
||
|
|
"rewards/rejected": -0.38164117670214526,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 14.853438377380371,
|
||
|
|
"kl": 13.6441068649292,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -35015680.0,
|
||
|
|
"logits/rejected": -35233779.07886435,
|
||
|
|
"logps/chosen": -475.5483746130031,
|
||
|
|
"logps/rejected": -401.3035291798107,
|
||
|
|
"loss": 0.5531,
|
||
|
|
"loss/base_kto": 3.147824764251709,
|
||
|
|
"metrics/advantage_var": 412.8899841308594,
|
||
|
|
"regularization/lipschitz_norm": 1291.1871337890625,
|
||
|
|
"regularization/mmd": 0.1924455761909485,
|
||
|
|
"regularization/rkhs_norm": 229.1018829345703,
|
||
|
|
"regularization/w1": 1.0845972299575806,
|
||
|
|
"rewards/chosen": 0.566196642423931,
|
||
|
|
"rewards/margins": 0.8774657828178831,
|
||
|
|
"rewards/rejected": -0.3112691403939521,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 20.79256248474121,
|
||
|
|
"kl": 12.546652793884277,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -34735211.627627626,
|
||
|
|
"logits/rejected": -35136301.863192186,
|
||
|
|
"logps/chosen": -478.5987237237237,
|
||
|
|
"logps/rejected": -376.67304560260584,
|
||
|
|
"loss": 0.562,
|
||
|
|
"loss/base_kto": 3.244504928588867,
|
||
|
|
"metrics/advantage_var": 393.84033203125,
|
||
|
|
"regularization/lipschitz_norm": 1269.017822265625,
|
||
|
|
"regularization/mmd": 0.1853182017803192,
|
||
|
|
"regularization/rkhs_norm": 220.6168975830078,
|
||
|
|
"regularization/w1": 1.0659749507904053,
|
||
|
|
"rewards/chosen": 0.49982431772592906,
|
||
|
|
"rewards/margins": 0.7822574454615765,
|
||
|
|
"rewards/rejected": -0.2824331277356474,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 15.379985809326172,
|
||
|
|
"kl": 15.253310203552246,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -33947831.5008,
|
||
|
|
"logits/rejected": -35959543.792366415,
|
||
|
|
"logps/chosen": -502.18095,
|
||
|
|
"logps/rejected": -373.40300572519084,
|
||
|
|
"loss": 0.5494,
|
||
|
|
"loss/base_kto": 3.17261004447937,
|
||
|
|
"metrics/advantage_var": 368.9141540527344,
|
||
|
|
"regularization/lipschitz_norm": 1238.4517822265625,
|
||
|
|
"regularization/mmd": 0.18214094638824463,
|
||
|
|
"regularization/rkhs_norm": 216.83447265625,
|
||
|
|
"regularization/w1": 1.040299415588379,
|
||
|
|
"rewards/chosen": 0.50336171875,
|
||
|
|
"rewards/margins": 0.8354358797262643,
|
||
|
|
"rewards/rejected": -0.3320741609762643,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 19.39569664001465,
|
||
|
|
"kl": 13.712936401367188,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -35031392.415584415,
|
||
|
|
"logits/rejected": -36158874.21686747,
|
||
|
|
"logps/chosen": -506.0470779220779,
|
||
|
|
"logps/rejected": -370.9024614081325,
|
||
|
|
"loss": 0.5609,
|
||
|
|
"loss/base_kto": 3.2079620361328125,
|
||
|
|
"metrics/advantage_var": 394.97760009765625,
|
||
|
|
"regularization/lipschitz_norm": 1299.3316650390625,
|
||
|
|
"regularization/mmd": 0.18789111077785492,
|
||
|
|
"regularization/rkhs_norm": 223.6798858642578,
|
||
|
|
"regularization/w1": 1.0914386510849,
|
||
|
|
"rewards/chosen": 0.5051978470443131,
|
||
|
|
"rewards/margins": 0.8047521892912305,
|
||
|
|
"rewards/rejected": -0.29955434224691735,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 13.761805534362793,
|
||
|
|
"kl": 11.173375129699707,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -34874392.11302983,
|
||
|
|
"logits/rejected": -36696140.44167963,
|
||
|
|
"logps/chosen": -479.92695251177395,
|
||
|
|
"logps/rejected": -380.33918157076204,
|
||
|
|
"loss": 0.5564,
|
||
|
|
"loss/base_kto": 3.2114906311035156,
|
||
|
|
"metrics/advantage_var": 413.20562744140625,
|
||
|
|
"regularization/lipschitz_norm": 1248.465576171875,
|
||
|
|
"regularization/mmd": 0.1912769228219986,
|
||
|
|
"regularization/rkhs_norm": 227.7106170654297,
|
||
|
|
"regularization/w1": 1.0487110614776611,
|
||
|
|
"rewards/chosen": 0.4889494747718799,
|
||
|
|
"rewards/margins": 0.8360280740046024,
|
||
|
|
"rewards/rejected": -0.3470785992327226,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 17.654193878173828,
|
||
|
|
"kl": 13.366930961608887,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -34288072.5815832,
|
||
|
|
"logits/rejected": -34046082.1301059,
|
||
|
|
"logps/chosen": -464.97354604200325,
|
||
|
|
"logps/rejected": -386.4766215960666,
|
||
|
|
"loss": 0.5481,
|
||
|
|
"loss/base_kto": 3.17956805229187,
|
||
|
|
"metrics/advantage_var": 395.4222412109375,
|
||
|
|
"regularization/lipschitz_norm": 1211.761962890625,
|
||
|
|
"regularization/mmd": 0.18699140846729279,
|
||
|
|
"regularization/rkhs_norm": 222.6088104248047,
|
||
|
|
"regularization/w1": 1.0178799629211426,
|
||
|
|
"rewards/chosen": 0.4826929341995658,
|
||
|
|
"rewards/margins": 0.838417182349859,
|
||
|
|
"rewards/rejected": -0.35572424815029313,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 17.65015983581543,
|
||
|
|
"kl": 10.807543754577637,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -33634580.75675676,
|
||
|
|
"logits/rejected": -35683564.82084691,
|
||
|
|
"logps/chosen": -486.1567192192192,
|
||
|
|
"logps/rejected": -353.6041327361564,
|
||
|
|
"loss": 0.5573,
|
||
|
|
"loss/base_kto": 3.2268807888031006,
|
||
|
|
"metrics/advantage_var": 364.9976501464844,
|
||
|
|
"regularization/lipschitz_norm": 1247.8719482421875,
|
||
|
|
"regularization/mmd": 0.18321049213409424,
|
||
|
|
"regularization/rkhs_norm": 218.1077423095703,
|
||
|
|
"regularization/w1": 1.0482124090194702,
|
||
|
|
"rewards/chosen": 0.48492912773613456,
|
||
|
|
"rewards/margins": 0.8114010839705477,
|
||
|
|
"rewards/rejected": -0.32647195623441316,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 12.414267539978027,
|
||
|
|
"kl": 10.244199752807617,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34719289.94728682,
|
||
|
|
"logits/rejected": -35485766.954330705,
|
||
|
|
"logps/chosen": -495.74588178294573,
|
||
|
|
"logps/rejected": -365.51343503937005,
|
||
|
|
"loss": 0.5593,
|
||
|
|
"loss/base_kto": 3.2109146118164062,
|
||
|
|
"metrics/advantage_var": 393.6255798339844,
|
||
|
|
"regularization/lipschitz_norm": 1279.9583740234375,
|
||
|
|
"regularization/mmd": 0.188627228140831,
|
||
|
|
"regularization/rkhs_norm": 224.5562286376953,
|
||
|
|
"regularization/w1": 1.075165033340454,
|
||
|
|
"rewards/chosen": 0.46203821463178296,
|
||
|
|
"rewards/margins": 0.8234383803398293,
|
||
|
|
"rewards/rejected": -0.3614001657080463,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 18.833457946777344,
|
||
|
|
"kl": 11.212935447692871,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -34887413.85736925,
|
||
|
|
"logits/rejected": -35249727.901386745,
|
||
|
|
"logps/chosen": -476.77639659271,
|
||
|
|
"logps/rejected": -381.8637085901387,
|
||
|
|
"loss": 0.5588,
|
||
|
|
"loss/base_kto": 3.2095658779144287,
|
||
|
|
"metrics/advantage_var": 417.80645751953125,
|
||
|
|
"regularization/lipschitz_norm": 1273.4381103515625,
|
||
|
|
"regularization/mmd": 0.19097180664539337,
|
||
|
|
"regularization/rkhs_norm": 227.34739685058594,
|
||
|
|
"regularization/w1": 1.069688081741333,
|
||
|
|
"rewards/chosen": 0.5112273734647385,
|
||
|
|
"rewards/margins": 0.8322144577109345,
|
||
|
|
"rewards/rejected": -0.3209870842461961,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 20.669588088989258,
|
||
|
|
"kl": 11.685368537902832,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -34633759.70278638,
|
||
|
|
"logits/rejected": -35840730.04416404,
|
||
|
|
"logps/chosen": -491.88351393188856,
|
||
|
|
"logps/rejected": -371.67517744479494,
|
||
|
|
"loss": 0.563,
|
||
|
|
"loss/base_kto": 3.1795499324798584,
|
||
|
|
"metrics/advantage_var": 433.4483337402344,
|
||
|
|
"regularization/lipschitz_norm": 1341.7772216796875,
|
||
|
|
"regularization/mmd": 0.1975117325782776,
|
||
|
|
"regularization/rkhs_norm": 235.1330108642578,
|
||
|
|
"regularization/w1": 1.127092957496643,
|
||
|
|
"rewards/chosen": 0.49785611651630224,
|
||
|
|
"rewards/margins": 0.8596400075864545,
|
||
|
|
"rewards/rejected": -0.3617838910701523,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 16.836538314819336,
|
||
|
|
"kl": 10.32569694519043,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -35831767.16564417,
|
||
|
|
"logits/rejected": -37297116.12738854,
|
||
|
|
"logps/chosen": -498.51447469325154,
|
||
|
|
"logps/rejected": -398.445113455414,
|
||
|
|
"loss": 0.5689,
|
||
|
|
"loss/base_kto": 3.2477729320526123,
|
||
|
|
"metrics/advantage_var": 443.9922790527344,
|
||
|
|
"regularization/lipschitz_norm": 1318.6453857421875,
|
||
|
|
"regularization/mmd": 0.19599764049053192,
|
||
|
|
"regularization/rkhs_norm": 233.33056640625,
|
||
|
|
"regularization/w1": 1.1076620817184448,
|
||
|
|
"rewards/chosen": 0.4396920233416411,
|
||
|
|
"rewards/margins": 0.8148518518678403,
|
||
|
|
"rewards/rejected": -0.3751598285261992,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 14.506603240966797,
|
||
|
|
"kl": 9.835335731506348,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -34089087.80980684,
|
||
|
|
"logits/rejected": -35805781.192751236,
|
||
|
|
"logps/chosen": -453.4229197622585,
|
||
|
|
"logps/rejected": -373.2368719110379,
|
||
|
|
"loss": 0.5564,
|
||
|
|
"loss/base_kto": 3.2597196102142334,
|
||
|
|
"metrics/advantage_var": 369.1478576660156,
|
||
|
|
"regularization/lipschitz_norm": 1205.0096435546875,
|
||
|
|
"regularization/mmd": 0.17931826412677765,
|
||
|
|
"regularization/rkhs_norm": 213.47412109375,
|
||
|
|
"regularization/w1": 1.0122079849243164,
|
||
|
|
"rewards/chosen": 0.4370893503900446,
|
||
|
|
"rewards/margins": 0.778400357391692,
|
||
|
|
"rewards/rejected": -0.34131100700164746,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 22.076805114746094,
|
||
|
|
"kl": 9.519233703613281,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -33823460.356807515,
|
||
|
|
"logits/rejected": -35462068.91731669,
|
||
|
|
"logps/chosen": -487.1495500782473,
|
||
|
|
"logps/rejected": -370.08097698907955,
|
||
|
|
"loss": 0.5663,
|
||
|
|
"loss/base_kto": 3.1691582202911377,
|
||
|
|
"metrics/advantage_var": 441.5194396972656,
|
||
|
|
"regularization/lipschitz_norm": 1383.5662841796875,
|
||
|
|
"regularization/mmd": 0.19881601631641388,
|
||
|
|
"regularization/rkhs_norm": 236.68569946289062,
|
||
|
|
"regularization/w1": 1.1621955633163452,
|
||
|
|
"rewards/chosen": 0.52571757858348,
|
||
|
|
"rewards/margins": 0.8854318902516498,
|
||
|
|
"rewards/rejected": -0.35971431166816986,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 14.2496919631958,
|
||
|
|
"kl": 13.717219352722168,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -34295744.933549434,
|
||
|
|
"logits/rejected": -34580705.49622926,
|
||
|
|
"logps/chosen": -497.1253038897893,
|
||
|
|
"logps/rejected": -380.0371889140271,
|
||
|
|
"loss": 0.5569,
|
||
|
|
"loss/base_kto": 3.1889755725860596,
|
||
|
|
"metrics/advantage_var": 387.796630859375,
|
||
|
|
"regularization/lipschitz_norm": 1285.385498046875,
|
||
|
|
"regularization/mmd": 0.18668226897716522,
|
||
|
|
"regularization/rkhs_norm": 222.24082946777344,
|
||
|
|
"regularization/w1": 1.0797237157821655,
|
||
|
|
"rewards/chosen": 0.4940602999645462,
|
||
|
|
"rewards/margins": 0.8334481044590905,
|
||
|
|
"rewards/rejected": -0.3393878044945442,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 13.872313499450684,
|
||
|
|
"kl": 12.012479782104492,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -35815178.99386503,
|
||
|
|
"logits/rejected": -36144154.089171976,
|
||
|
|
"logps/chosen": -469.13990605828224,
|
||
|
|
"logps/rejected": -391.9139380971338,
|
||
|
|
"loss": 0.5527,
|
||
|
|
"loss/base_kto": 3.1606357097625732,
|
||
|
|
"metrics/advantage_var": 411.4314880371094,
|
||
|
|
"regularization/lipschitz_norm": 1273.3734130859375,
|
||
|
|
"regularization/mmd": 0.1916976422071457,
|
||
|
|
"regularization/rkhs_norm": 228.2114715576172,
|
||
|
|
"regularization/w1": 1.0696337223052979,
|
||
|
|
"rewards/chosen": 0.48192573032496167,
|
||
|
|
"rewards/margins": 0.8695787166935176,
|
||
|
|
"rewards/rejected": -0.38765298636855594,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.98294195064275e+17,
|
||
|
|
"train_loss": 0.5869958709558675,
|
||
|
|
"train_runtime": 11115.9487,
|
||
|
|
"train_samples_per_second": 13.334,
|
||
|
|
"train_steps_per_second": 0.208
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.98294195064275e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|