2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 21.30620574951172,
|
||
|
|
"kl": 16.385543823242188,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37237433.32515337,
|
||
|
|
"logits/rejected": -38183799.031847134,
|
||
|
|
"logps/chosen": -458.3118769171779,
|
||
|
|
"logps/rejected": -380.0686703821656,
|
||
|
|
"loss": 0.5989,
|
||
|
|
"loss/base_kto": 3.887625217437744,
|
||
|
|
"metrics/advantage_var": 191.3936004638672,
|
||
|
|
"regularization/lipschitz_norm": 919.4276733398438,
|
||
|
|
"regularization/mmd": 0.13125856220722198,
|
||
|
|
"regularization/rkhs_norm": 156.2601776123047,
|
||
|
|
"regularization/w1": 0.7723193168640137,
|
||
|
|
"rewards/chosen": 0.2465540412013516,
|
||
|
|
"rewards/margins": 0.08952261140033399,
|
||
|
|
"rewards/rejected": 0.1570314298010176,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 23.404104232788086,
|
||
|
|
"kl": 9.088505744934082,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -36791531.810725555,
|
||
|
|
"logits/rejected": -37880429.374613,
|
||
|
|
"logps/chosen": -479.285636829653,
|
||
|
|
"logps/rejected": -363.20839783281735,
|
||
|
|
"loss": 0.604,
|
||
|
|
"loss/base_kto": 3.9080753326416016,
|
||
|
|
"metrics/advantage_var": 237.0071258544922,
|
||
|
|
"regularization/lipschitz_norm": 941.30517578125,
|
||
|
|
"regularization/mmd": 0.1333685964345932,
|
||
|
|
"regularization/rkhs_norm": 158.7721405029297,
|
||
|
|
"regularization/w1": 0.790696382522583,
|
||
|
|
"rewards/chosen": 0.1017112852271047,
|
||
|
|
"rewards/margins": 0.09241618113830667,
|
||
|
|
"rewards/rejected": 0.009295104088798026,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 21.467105865478516,
|
||
|
|
"kl": 11.46056079864502,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -35823341.92941176,
|
||
|
|
"logits/rejected": -37332244.48,
|
||
|
|
"logps/chosen": -463.5670955882353,
|
||
|
|
"logps/rejected": -382.3347395833333,
|
||
|
|
"loss": 0.6039,
|
||
|
|
"loss/base_kto": 3.8919856548309326,
|
||
|
|
"metrics/advantage_var": 192.63877868652344,
|
||
|
|
"regularization/lipschitz_norm": 956.9761962890625,
|
||
|
|
"regularization/mmd": 0.1353202611207962,
|
||
|
|
"regularization/rkhs_norm": 161.09556579589844,
|
||
|
|
"regularization/w1": 0.8038601279258728,
|
||
|
|
"rewards/chosen": 0.18667184044333066,
|
||
|
|
"rewards/margins": 0.10561023562562233,
|
||
|
|
"rewards/rejected": 0.08106160481770833,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 22.6914005279541,
|
||
|
|
"kl": 8.611804008483887,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -36030081.83732057,
|
||
|
|
"logits/rejected": -37339907.52833078,
|
||
|
|
"logps/chosen": -487.0245215311005,
|
||
|
|
"logps/rejected": -385.3145817381317,
|
||
|
|
"loss": 0.5973,
|
||
|
|
"loss/base_kto": 3.8769938945770264,
|
||
|
|
"metrics/advantage_var": 182.3870849609375,
|
||
|
|
"regularization/lipschitz_norm": 915.8214721679688,
|
||
|
|
"regularization/mmd": 0.13175193965435028,
|
||
|
|
"regularization/rkhs_norm": 156.84756469726562,
|
||
|
|
"regularization/w1": 0.7692900896072388,
|
||
|
|
"rewards/chosen": 0.08962506502629088,
|
||
|
|
"rewards/margins": 0.10581642927372834,
|
||
|
|
"rewards/rejected": -0.01619136424743746,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 23.67812728881836,
|
||
|
|
"kl": 16.48870086669922,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -37817934.88821752,
|
||
|
|
"logits/rejected": -37920091.96116505,
|
||
|
|
"logps/chosen": -480.29966012084594,
|
||
|
|
"logps/rejected": -369.67157160194176,
|
||
|
|
"loss": 0.5939,
|
||
|
|
"loss/base_kto": 3.8202621936798096,
|
||
|
|
"metrics/advantage_var": 204.16806030273438,
|
||
|
|
"regularization/lipschitz_norm": 944.6422729492188,
|
||
|
|
"regularization/mmd": 0.1374884694814682,
|
||
|
|
"regularization/rkhs_norm": 163.6767578125,
|
||
|
|
"regularization/w1": 0.7934995889663696,
|
||
|
|
"rewards/chosen": 0.3411060805767324,
|
||
|
|
"rewards/margins": 0.14616479136016308,
|
||
|
|
"rewards/rejected": 0.19494128921656934,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 21.86313819885254,
|
||
|
|
"kl": 13.873870849609375,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -36919010.6749226,
|
||
|
|
"logits/rejected": -36583236.643533126,
|
||
|
|
"logps/chosen": -469.7824109907121,
|
||
|
|
"logps/rejected": -343.7594390772871,
|
||
|
|
"loss": 0.5917,
|
||
|
|
"loss/base_kto": 3.8518707752227783,
|
||
|
|
"metrics/advantage_var": 185.32675170898438,
|
||
|
|
"regularization/lipschitz_norm": 897.4934692382812,
|
||
|
|
"regularization/mmd": 0.12806062400341034,
|
||
|
|
"regularization/rkhs_norm": 152.45315551757812,
|
||
|
|
"regularization/w1": 0.7538945078849792,
|
||
|
|
"rewards/chosen": 0.2243381417572683,
|
||
|
|
"rewards/margins": 0.15287296539454773,
|
||
|
|
"rewards/rejected": 0.07146517636272057,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 24.354427337646484,
|
||
|
|
"kl": 5.70906400680542,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -36384878.3518225,
|
||
|
|
"logits/rejected": -37398903.5192604,
|
||
|
|
"logps/chosen": -474.1553090332805,
|
||
|
|
"logps/rejected": -379.6500385208012,
|
||
|
|
"loss": 0.5914,
|
||
|
|
"loss/base_kto": 3.8447353839874268,
|
||
|
|
"metrics/advantage_var": 179.997314453125,
|
||
|
|
"regularization/lipschitz_norm": 901.4913940429688,
|
||
|
|
"regularization/mmd": 0.1292407363653183,
|
||
|
|
"regularization/rkhs_norm": 153.85801696777344,
|
||
|
|
"regularization/w1": 0.7572528123855591,
|
||
|
|
"rewards/chosen": 0.08524320091574014,
|
||
|
|
"rewards/margins": 0.14999670540582105,
|
||
|
|
"rewards/rejected": -0.0647535044900809,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 24.62518310546875,
|
||
|
|
"kl": 7.673250675201416,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36190016.81012658,
|
||
|
|
"logits/rejected": -37570857.086419754,
|
||
|
|
"logps/chosen": -490.16307357594934,
|
||
|
|
"logps/rejected": -368.49573206018516,
|
||
|
|
"loss": 0.5996,
|
||
|
|
"loss/base_kto": 3.8244552612304688,
|
||
|
|
"metrics/advantage_var": 207.548583984375,
|
||
|
|
"regularization/lipschitz_norm": 994.6609497070312,
|
||
|
|
"regularization/mmd": 0.1368967741727829,
|
||
|
|
"regularization/rkhs_norm": 162.97235107421875,
|
||
|
|
"regularization/w1": 0.8355152010917664,
|
||
|
|
"rewards/chosen": 0.1482733955866174,
|
||
|
|
"rewards/margins": 0.17031480321885648,
|
||
|
|
"rewards/rejected": -0.022041407632239073,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 18.731962203979492,
|
||
|
|
"kl": 7.896986484527588,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -36152657.01265823,
|
||
|
|
"logits/rejected": -38387958.518518515,
|
||
|
|
"logps/chosen": -520.5198773734177,
|
||
|
|
"logps/rejected": -395.7608748070988,
|
||
|
|
"loss": 0.6026,
|
||
|
|
"loss/base_kto": 3.7690110206604004,
|
||
|
|
"metrics/advantage_var": 246.31382751464844,
|
||
|
|
"regularization/lipschitz_norm": 1073.66796875,
|
||
|
|
"regularization/mmd": 0.14970853924751282,
|
||
|
|
"regularization/rkhs_norm": 178.22445678710938,
|
||
|
|
"regularization/w1": 0.901881217956543,
|
||
|
|
"rewards/chosen": 0.08006769494165349,
|
||
|
|
"rewards/margins": 0.2120290465011394,
|
||
|
|
"rewards/rejected": -0.1319613515594859,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 21.32183265686035,
|
||
|
|
"kl": 9.421449661254883,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -36843970.03821656,
|
||
|
|
"logits/rejected": -37945268.61349693,
|
||
|
|
"logps/chosen": -519.3416600318471,
|
||
|
|
"logps/rejected": -361.66645897239266,
|
||
|
|
"loss": 0.5974,
|
||
|
|
"loss/base_kto": 3.828326463699341,
|
||
|
|
"metrics/advantage_var": 211.9000701904297,
|
||
|
|
"regularization/lipschitz_norm": 965.6625366210938,
|
||
|
|
"regularization/mmd": 0.13956843316555023,
|
||
|
|
"regularization/rkhs_norm": 166.1528778076172,
|
||
|
|
"regularization/w1": 0.8111564517021179,
|
||
|
|
"rewards/chosen": 0.10966109014620447,
|
||
|
|
"rewards/margins": 0.16354339063426104,
|
||
|
|
"rewards/rejected": -0.05388230048805658,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 22.96819496154785,
|
||
|
|
"kl": 3.105022430419922,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37329087.18987342,
|
||
|
|
"logits/rejected": -37404877.43209877,
|
||
|
|
"logps/chosen": -473.15644778481015,
|
||
|
|
"logps/rejected": -414.8044945987654,
|
||
|
|
"loss": 0.5992,
|
||
|
|
"loss/base_kto": 3.8584632873535156,
|
||
|
|
"metrics/advantage_var": 207.459716796875,
|
||
|
|
"regularization/lipschitz_norm": 950.4669189453125,
|
||
|
|
"regularization/mmd": 0.1365087777376175,
|
||
|
|
"regularization/rkhs_norm": 162.5104522705078,
|
||
|
|
"regularization/w1": 0.7983922362327576,
|
||
|
|
"rewards/chosen": -0.08360711834098719,
|
||
|
|
"rewards/margins": 0.13516261447573252,
|
||
|
|
"rewards/rejected": -0.21876973281671971,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 28.522031784057617,
|
||
|
|
"kl": 11.790318489074707,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -35376852.45151033,
|
||
|
|
"logits/rejected": -34778815.115207374,
|
||
|
|
"logps/chosen": -492.73400238473766,
|
||
|
|
"logps/rejected": -388.49961597542244,
|
||
|
|
"loss": 0.5992,
|
||
|
|
"loss/base_kto": 3.7751381397247314,
|
||
|
|
"metrics/advantage_var": 243.459228515625,
|
||
|
|
"regularization/lipschitz_norm": 1035.931396484375,
|
||
|
|
"regularization/mmd": 0.14857906103134155,
|
||
|
|
"regularization/rkhs_norm": 176.8798370361328,
|
||
|
|
"regularization/w1": 0.870182454586029,
|
||
|
|
"rewards/chosen": 0.1879951464921378,
|
||
|
|
"rewards/margins": 0.21998687950992596,
|
||
|
|
"rewards/rejected": -0.03199173301778814,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 26.046768188476562,
|
||
|
|
"kl": 16.253116607666016,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -36735619.83233533,
|
||
|
|
"logits/rejected": -37659105.88235294,
|
||
|
|
"logps/chosen": -476.267870508982,
|
||
|
|
"logps/rejected": -351.4927236519608,
|
||
|
|
"loss": 0.6088,
|
||
|
|
"loss/base_kto": 3.823444366455078,
|
||
|
|
"metrics/advantage_var": 260.6925964355469,
|
||
|
|
"regularization/lipschitz_norm": 1064.2943115234375,
|
||
|
|
"regularization/mmd": 0.15288196504116058,
|
||
|
|
"regularization/rkhs_norm": 182.00230407714844,
|
||
|
|
"regularization/w1": 0.8940073251724243,
|
||
|
|
"rewards/chosen": 0.2839615873234001,
|
||
|
|
"rewards/margins": 0.1419578485211572,
|
||
|
|
"rewards/rejected": 0.1420037388022429,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 22.29317855834961,
|
||
|
|
"kl": 18.77423667907715,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -37005181.67272727,
|
||
|
|
"logits/rejected": -37470330.21935484,
|
||
|
|
"logps/chosen": -485.25710227272725,
|
||
|
|
"logps/rejected": -359.2376008064516,
|
||
|
|
"loss": 0.5959,
|
||
|
|
"loss/base_kto": 3.746065616607666,
|
||
|
|
"metrics/advantage_var": 290.56085205078125,
|
||
|
|
"regularization/lipschitz_norm": 1033.027587890625,
|
||
|
|
"regularization/mmd": 0.15310192108154297,
|
||
|
|
"regularization/rkhs_norm": 182.26419067382812,
|
||
|
|
"regularization/w1": 0.8677431344985962,
|
||
|
|
"rewards/chosen": 0.3154003952488755,
|
||
|
|
"rewards/margins": 0.2193312069770761,
|
||
|
|
"rewards/rejected": 0.09606918827179939,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 17.760087966918945,
|
||
|
|
"kl": 5.830892086029053,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -36442131.91572123,
|
||
|
|
"logits/rejected": -36848749.65912519,
|
||
|
|
"logps/chosen": -496.06624797406806,
|
||
|
|
"logps/rejected": -370.4114347662142,
|
||
|
|
"loss": 0.5979,
|
||
|
|
"loss/base_kto": 3.778766632080078,
|
||
|
|
"metrics/advantage_var": 234.72276306152344,
|
||
|
|
"regularization/lipschitz_norm": 1019.4176025390625,
|
||
|
|
"regularization/mmd": 0.14803357422351837,
|
||
|
|
"regularization/rkhs_norm": 176.23045349121094,
|
||
|
|
"regularization/w1": 0.8563107848167419,
|
||
|
|
"rewards/chosen": 0.028018224954218688,
|
||
|
|
"rewards/margins": 0.2139656834429477,
|
||
|
|
"rewards/rejected": -0.185947458488729,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 27.020835876464844,
|
||
|
|
"kl": 12.562323570251465,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -36587044.864,
|
||
|
|
"logits/rejected": -37103853.63053435,
|
||
|
|
"logps/chosen": -466.8468,
|
||
|
|
"logps/rejected": -394.52194656488547,
|
||
|
|
"loss": 0.605,
|
||
|
|
"loss/base_kto": 3.8280041217803955,
|
||
|
|
"metrics/advantage_var": 261.6440124511719,
|
||
|
|
"regularization/lipschitz_norm": 1030.7325439453125,
|
||
|
|
"regularization/mmd": 0.14636702835559845,
|
||
|
|
"regularization/rkhs_norm": 174.2464599609375,
|
||
|
|
"regularization/w1": 0.8658153414726257,
|
||
|
|
"rewards/chosen": 0.2149145263671875,
|
||
|
|
"rewards/margins": 0.15796180671517174,
|
||
|
|
"rewards/rejected": 0.056952719652015746,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 19.868404388427734,
|
||
|
|
"kl": 14.786349296569824,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -37221587.699846864,
|
||
|
|
"logits/rejected": -37740988.22328549,
|
||
|
|
"logps/chosen": -465.51124617151606,
|
||
|
|
"logps/rejected": -361.75343899521533,
|
||
|
|
"loss": 0.5978,
|
||
|
|
"loss/base_kto": 3.7788236141204834,
|
||
|
|
"metrics/advantage_var": 236.5540008544922,
|
||
|
|
"regularization/lipschitz_norm": 1020.3147583007812,
|
||
|
|
"regularization/mmd": 0.1464555859565735,
|
||
|
|
"regularization/rkhs_norm": 174.35189819335938,
|
||
|
|
"regularization/w1": 0.857064425945282,
|
||
|
|
"rewards/chosen": 0.2893020139171372,
|
||
|
|
"rewards/margins": 0.19903800230897084,
|
||
|
|
"rewards/rejected": 0.09026401160816637,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 29.049692153930664,
|
||
|
|
"kl": 4.6120924949646,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -36731300.72665535,
|
||
|
|
"logits/rejected": -37294133.348769896,
|
||
|
|
"logps/chosen": -508.23418930390494,
|
||
|
|
"logps/rejected": -369.49638205499275,
|
||
|
|
"loss": 0.5931,
|
||
|
|
"loss/base_kto": 3.7556424140930176,
|
||
|
|
"metrics/advantage_var": 233.97671508789062,
|
||
|
|
"regularization/lipschitz_norm": 1001.9880981445312,
|
||
|
|
"regularization/mmd": 0.14747083187103271,
|
||
|
|
"regularization/rkhs_norm": 175.56053161621094,
|
||
|
|
"regularization/w1": 0.841670036315918,
|
||
|
|
"rewards/chosen": -0.03298977073060636,
|
||
|
|
"rewards/margins": 0.21002983012243723,
|
||
|
|
"rewards/rejected": -0.24301960085304358,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 16.215953826904297,
|
||
|
|
"kl": 5.461564540863037,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -36202803.869281046,
|
||
|
|
"logits/rejected": -36440959.233532935,
|
||
|
|
"logps/chosen": -453.9362745098039,
|
||
|
|
"logps/rejected": -358.4580838323353,
|
||
|
|
"loss": 0.597,
|
||
|
|
"loss/base_kto": 3.7826297283172607,
|
||
|
|
"metrics/advantage_var": 239.4388427734375,
|
||
|
|
"regularization/lipschitz_norm": 1004.7897338867188,
|
||
|
|
"regularization/mmd": 0.14958620071411133,
|
||
|
|
"regularization/rkhs_norm": 178.0788116455078,
|
||
|
|
"regularization/w1": 0.8440233469009399,
|
||
|
|
"rewards/chosen": -0.11870555005042381,
|
||
|
|
"rewards/margins": 0.1800786945097722,
|
||
|
|
"rewards/rejected": -0.29878424456019603,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 24.50790786743164,
|
||
|
|
"kl": 1.323299527168274,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -35323897.456869006,
|
||
|
|
"logits/rejected": -36453892.69724771,
|
||
|
|
"logps/chosen": -481.3480431309904,
|
||
|
|
"logps/rejected": -379.06751720183485,
|
||
|
|
"loss": 0.5986,
|
||
|
|
"loss/base_kto": 3.7839763164520264,
|
||
|
|
"metrics/advantage_var": 236.5087890625,
|
||
|
|
"regularization/lipschitz_norm": 1018.1357421875,
|
||
|
|
"regularization/mmd": 0.14926938712596893,
|
||
|
|
"regularization/rkhs_norm": 177.70164489746094,
|
||
|
|
"regularization/w1": 0.8552339673042297,
|
||
|
|
"rewards/chosen": -0.2617264281446561,
|
||
|
|
"rewards/margins": 0.21538011534671914,
|
||
|
|
"rewards/rejected": -0.4771065434913752,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 20.045339584350586,
|
||
|
|
"kl": 4.559103488922119,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36375742.54765507,
|
||
|
|
"logits/rejected": -36945344.310177706,
|
||
|
|
"logps/chosen": -485.43792549167927,
|
||
|
|
"logps/rejected": -375.23434975767367,
|
||
|
|
"loss": 0.591,
|
||
|
|
"loss/base_kto": 3.755572557449341,
|
||
|
|
"metrics/advantage_var": 222.8901824951172,
|
||
|
|
"regularization/lipschitz_norm": 987.20556640625,
|
||
|
|
"regularization/mmd": 0.143472820520401,
|
||
|
|
"regularization/rkhs_norm": 170.8009796142578,
|
||
|
|
"regularization/w1": 0.8292526602745056,
|
||
|
|
"rewards/chosen": 0.08200028227605545,
|
||
|
|
"rewards/margins": 0.25845988282018534,
|
||
|
|
"rewards/rejected": -0.1764596005441299,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 18.75247573852539,
|
||
|
|
"kl": 4.5143818855285645,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -35564661.12418301,
|
||
|
|
"logits/rejected": -36188647.473053895,
|
||
|
|
"logps/chosen": -487.8157169117647,
|
||
|
|
"logps/rejected": -380.7473100673653,
|
||
|
|
"loss": 0.6056,
|
||
|
|
"loss/base_kto": 3.810009002685547,
|
||
|
|
"metrics/advantage_var": 253.3105926513672,
|
||
|
|
"regularization/lipschitz_norm": 1051.658447265625,
|
||
|
|
"regularization/mmd": 0.15111172199249268,
|
||
|
|
"regularization/rkhs_norm": 179.89492797851562,
|
||
|
|
"regularization/w1": 0.8833931088447571,
|
||
|
|
"rewards/chosen": -0.10365574655969159,
|
||
|
|
"rewards/margins": 0.147400673315402,
|
||
|
|
"rewards/rejected": -0.2510564198750936,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 32.510154724121094,
|
||
|
|
"kl": 7.408751010894775,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -35925008.15141956,
|
||
|
|
"logits/rejected": -37290427.839009285,
|
||
|
|
"logps/chosen": -495.5834976340694,
|
||
|
|
"logps/rejected": -358.63201431888547,
|
||
|
|
"loss": 0.5926,
|
||
|
|
"loss/base_kto": 3.683509111404419,
|
||
|
|
"metrics/advantage_var": 265.0987854003906,
|
||
|
|
"regularization/lipschitz_norm": 1074.9356689453125,
|
||
|
|
"regularization/mmd": 0.15402238070964813,
|
||
|
|
"regularization/rkhs_norm": 183.35997009277344,
|
||
|
|
"regularization/w1": 0.9029459357261658,
|
||
|
|
"rewards/chosen": 0.139697872125764,
|
||
|
|
"rewards/margins": 0.2905832503698865,
|
||
|
|
"rewards/rejected": -0.1508853782441225,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 20.378629684448242,
|
||
|
|
"kl": 4.379853248596191,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -36566377.50544324,
|
||
|
|
"logits/rejected": -36899237.17425432,
|
||
|
|
"logps/chosen": -467.98833592534993,
|
||
|
|
"logps/rejected": -379.43597919937207,
|
||
|
|
"loss": 0.6013,
|
||
|
|
"loss/base_kto": 3.7656960487365723,
|
||
|
|
"metrics/advantage_var": 267.1982421875,
|
||
|
|
"regularization/lipschitz_norm": 1059.279296875,
|
||
|
|
"regularization/mmd": 0.15457189083099365,
|
||
|
|
"regularization/rkhs_norm": 184.01417541503906,
|
||
|
|
"regularization/w1": 0.8897945284843445,
|
||
|
|
"rewards/chosen": -8.799796156341885e-05,
|
||
|
|
"rewards/margins": 0.22673953067609356,
|
||
|
|
"rewards/rejected": -0.22682752863765698,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 21.59541130065918,
|
||
|
|
"kl": 8.324943542480469,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36415978.73482428,
|
||
|
|
"logits/rejected": -38221222.75229358,
|
||
|
|
"logps/chosen": -507.32438099041536,
|
||
|
|
"logps/rejected": -362.50936544342505,
|
||
|
|
"loss": 0.5947,
|
||
|
|
"loss/base_kto": 3.7547378540039062,
|
||
|
|
"metrics/advantage_var": 231.9820098876953,
|
||
|
|
"regularization/lipschitz_norm": 1020.3601684570312,
|
||
|
|
"regularization/mmd": 0.14552414417266846,
|
||
|
|
"regularization/rkhs_norm": 173.24302673339844,
|
||
|
|
"regularization/w1": 0.8571025729179382,
|
||
|
|
"rewards/chosen": 0.04323362161557134,
|
||
|
|
"rewards/margins": 0.19958254709817838,
|
||
|
|
"rewards/rejected": -0.15634892548260704,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 20.982234954833984,
|
||
|
|
"kl": 5.086883068084717,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -34935692.41335453,
|
||
|
|
"logits/rejected": -35675260.26420891,
|
||
|
|
"logps/chosen": -494.2798589030207,
|
||
|
|
"logps/rejected": -365.8594950076805,
|
||
|
|
"loss": 0.6067,
|
||
|
|
"loss/base_kto": 3.7751259803771973,
|
||
|
|
"metrics/advantage_var": 269.0093688964844,
|
||
|
|
"regularization/lipschitz_norm": 1094.8719482421875,
|
||
|
|
"regularization/mmd": 0.15867407619953156,
|
||
|
|
"regularization/rkhs_norm": 188.897705078125,
|
||
|
|
"regularization/w1": 0.9196924567222595,
|
||
|
|
"rewards/chosen": -0.06926192388246473,
|
||
|
|
"rewards/margins": 0.22349587511667124,
|
||
|
|
"rewards/rejected": -0.29275779899913595,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 20.61083984375,
|
||
|
|
"kl": 5.628817558288574,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -37833826.99207607,
|
||
|
|
"logits/rejected": -38360889.19568567,
|
||
|
|
"logps/chosen": -504.9411152931854,
|
||
|
|
"logps/rejected": -371.56091101694915,
|
||
|
|
"loss": 0.5907,
|
||
|
|
"loss/base_kto": 3.678312063217163,
|
||
|
|
"metrics/advantage_var": 269.90863037109375,
|
||
|
|
"regularization/lipschitz_norm": 1062.9803466796875,
|
||
|
|
"regularization/mmd": 0.15461526811122894,
|
||
|
|
"regularization/rkhs_norm": 184.0657958984375,
|
||
|
|
"regularization/w1": 0.8929035067558289,
|
||
|
|
"rewards/chosen": 0.06762640903188763,
|
||
|
|
"rewards/margins": 0.3164247500041767,
|
||
|
|
"rewards/rejected": -0.2487983409722891,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 18.963178634643555,
|
||
|
|
"kl": 6.945270538330078,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -35717953.41324921,
|
||
|
|
"logits/rejected": -37219115.591331266,
|
||
|
|
"logps/chosen": -472.8264491324921,
|
||
|
|
"logps/rejected": -363.4963960913313,
|
||
|
|
"loss": 0.5972,
|
||
|
|
"loss/base_kto": 3.7133948802948,
|
||
|
|
"metrics/advantage_var": 272.83770751953125,
|
||
|
|
"regularization/lipschitz_norm": 1079.7265625,
|
||
|
|
"regularization/mmd": 0.1568920910358429,
|
||
|
|
"regularization/rkhs_norm": 186.77630615234375,
|
||
|
|
"regularization/w1": 0.9069703221321106,
|
||
|
|
"rewards/chosen": 0.03154011554898524,
|
||
|
|
"rewards/margins": 0.2772439869541468,
|
||
|
|
"rewards/rejected": -0.24570387140516156,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 21.301929473876953,
|
||
|
|
"kl": 7.833394527435303,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36862122.398744114,
|
||
|
|
"logits/rejected": -37882766.93001555,
|
||
|
|
"logps/chosen": -471.8631279434851,
|
||
|
|
"logps/rejected": -384.9840347978227,
|
||
|
|
"loss": 0.6027,
|
||
|
|
"loss/base_kto": 3.8068490028381348,
|
||
|
|
"metrics/advantage_var": 243.0074920654297,
|
||
|
|
"regularization/lipschitz_norm": 1030.75439453125,
|
||
|
|
"regularization/mmd": 0.14884667098522186,
|
||
|
|
"regularization/rkhs_norm": 177.1984100341797,
|
||
|
|
"regularization/w1": 0.8658336997032166,
|
||
|
|
"rewards/chosen": 0.08991812761387608,
|
||
|
|
"rewards/margins": 0.19279751094427314,
|
||
|
|
"rewards/rejected": -0.10287938333039706,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 16.994449615478516,
|
||
|
|
"kl": 6.220043659210205,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -36866568.02507837,
|
||
|
|
"logits/rejected": -38472190.40498442,
|
||
|
|
"logps/chosen": -503.31612460815046,
|
||
|
|
"logps/rejected": -382.5021417445483,
|
||
|
|
"loss": 0.5947,
|
||
|
|
"loss/base_kto": 3.7638473510742188,
|
||
|
|
"metrics/advantage_var": 235.528564453125,
|
||
|
|
"regularization/lipschitz_norm": 1007.619140625,
|
||
|
|
"regularization/mmd": 0.1473793238401413,
|
||
|
|
"regularization/rkhs_norm": 175.45156860351562,
|
||
|
|
"regularization/w1": 0.8463999629020691,
|
||
|
|
"rewards/chosen": 0.01837258204397363,
|
||
|
|
"rewards/margins": 0.22212782707445697,
|
||
|
|
"rewards/rejected": -0.20375524503048334,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 22.941701889038086,
|
||
|
|
"kl": 9.876753807067871,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36347578.469135806,
|
||
|
|
"logits/rejected": -37466653.164556965,
|
||
|
|
"logps/chosen": -512.2447916666666,
|
||
|
|
"logps/rejected": -355.0952828322785,
|
||
|
|
"loss": 0.6013,
|
||
|
|
"loss/base_kto": 3.7260215282440186,
|
||
|
|
"metrics/advantage_var": 275.5116882324219,
|
||
|
|
"regularization/lipschitz_norm": 1103.078369140625,
|
||
|
|
"regularization/mmd": 0.15810923278331757,
|
||
|
|
"regularization/rkhs_norm": 188.2252655029297,
|
||
|
|
"regularization/w1": 0.9265857934951782,
|
||
|
|
"rewards/chosen": 0.10814003885528188,
|
||
|
|
"rewards/margins": 0.25599663047832255,
|
||
|
|
"rewards/rejected": -0.14785659162304068,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 22.692665100097656,
|
||
|
|
"kl": 12.700552940368652,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -35791884.3003003,
|
||
|
|
"logits/rejected": -36865420.92508143,
|
||
|
|
"logps/chosen": -476.85923423423424,
|
||
|
|
"logps/rejected": -370.5443556596091,
|
||
|
|
"loss": 0.592,
|
||
|
|
"loss/base_kto": 3.660593032836914,
|
||
|
|
"metrics/advantage_var": 277.190673828125,
|
||
|
|
"regularization/lipschitz_norm": 1092.2738037109375,
|
||
|
|
"regularization/mmd": 0.15797871351242065,
|
||
|
|
"regularization/rkhs_norm": 188.0699005126953,
|
||
|
|
"regularization/w1": 0.9175099730491638,
|
||
|
|
"rewards/chosen": 0.2714547738656625,
|
||
|
|
"rewards/margins": 0.2995656942778069,
|
||
|
|
"rewards/rejected": -0.028110920412144365,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 16.765060424804688,
|
||
|
|
"kl": 7.800302982330322,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36281006.71810089,
|
||
|
|
"logits/rejected": -36525944.81848185,
|
||
|
|
"logps/chosen": -473.14706973293767,
|
||
|
|
"logps/rejected": -382.9906662541254,
|
||
|
|
"loss": 0.607,
|
||
|
|
"loss/base_kto": 3.772947072982788,
|
||
|
|
"metrics/advantage_var": 256.4257507324219,
|
||
|
|
"regularization/lipschitz_norm": 1104.7586669921875,
|
||
|
|
"regularization/mmd": 0.1550176590681076,
|
||
|
|
"regularization/rkhs_norm": 184.5448455810547,
|
||
|
|
"regularization/w1": 0.9279972314834595,
|
||
|
|
"rewards/chosen": 0.14004827889915036,
|
||
|
|
"rewards/margins": 0.2160759381988938,
|
||
|
|
"rewards/rejected": -0.07602765929974345,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 19.784156799316406,
|
||
|
|
"kl": 18.611282348632812,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -36242914.96119403,
|
||
|
|
"logits/rejected": -36432563.619672135,
|
||
|
|
"logps/chosen": -487.7375932835821,
|
||
|
|
"logps/rejected": -374.94795081967214,
|
||
|
|
"loss": 0.5943,
|
||
|
|
"loss/base_kto": 3.7035744190216064,
|
||
|
|
"metrics/advantage_var": 255.98374938964844,
|
||
|
|
"regularization/lipschitz_norm": 1066.238525390625,
|
||
|
|
"regularization/mmd": 0.15486612915992737,
|
||
|
|
"regularization/rkhs_norm": 184.3644256591797,
|
||
|
|
"regularization/w1": 0.8956403732299805,
|
||
|
|
"rewards/chosen": 0.3499934182238223,
|
||
|
|
"rewards/margins": 0.24982007462791106,
|
||
|
|
"rewards/rejected": 0.10017334359591125,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 21.27347183227539,
|
||
|
|
"kl": 16.115873336791992,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -36517990.72100314,
|
||
|
|
"logits/rejected": -37601879.7258567,
|
||
|
|
"logps/chosen": -504.5242456896552,
|
||
|
|
"logps/rejected": -397.8655081775701,
|
||
|
|
"loss": 0.5956,
|
||
|
|
"loss/base_kto": 3.7432262897491455,
|
||
|
|
"metrics/advantage_var": 252.15293884277344,
|
||
|
|
"regularization/lipschitz_norm": 1038.50537109375,
|
||
|
|
"regularization/mmd": 0.14949001371860504,
|
||
|
|
"regularization/rkhs_norm": 177.96429443359375,
|
||
|
|
"regularization/w1": 0.8723444938659668,
|
||
|
|
"rewards/chosen": 0.2303431878642976,
|
||
|
|
"rewards/margins": 0.22849981562628105,
|
||
|
|
"rewards/rejected": 0.0018433722380165741,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 20.591951370239258,
|
||
|
|
"kl": 14.438664436340332,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -35569708.24691358,
|
||
|
|
"logits/rejected": -37203047.69620253,
|
||
|
|
"logps/chosen": -489.99455054012344,
|
||
|
|
"logps/rejected": -360.44986155063293,
|
||
|
|
"loss": 0.5936,
|
||
|
|
"loss/base_kto": 3.7016303539276123,
|
||
|
|
"metrics/advantage_var": 259.8722839355469,
|
||
|
|
"regularization/lipschitz_norm": 1064.8228759765625,
|
||
|
|
"regularization/mmd": 0.15258057415485382,
|
||
|
|
"regularization/rkhs_norm": 181.64353942871094,
|
||
|
|
"regularization/w1": 0.8944511413574219,
|
||
|
|
"rewards/chosen": 0.22748824696481965,
|
||
|
|
"rewards/margins": 0.2628883539018901,
|
||
|
|
"rewards/rejected": -0.035400106937070436,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 21.718122482299805,
|
||
|
|
"kl": 10.604936599731445,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36374055.384615384,
|
||
|
|
"logits/rejected": -35905230.048780486,
|
||
|
|
"logps/chosen": -485.7256610576923,
|
||
|
|
"logps/rejected": -383.9867568597561,
|
||
|
|
"loss": 0.6052,
|
||
|
|
"loss/base_kto": 3.7834489345550537,
|
||
|
|
"metrics/advantage_var": 255.31553649902344,
|
||
|
|
"regularization/lipschitz_norm": 1078.7470703125,
|
||
|
|
"regularization/mmd": 0.15181118249893188,
|
||
|
|
"regularization/rkhs_norm": 180.72760009765625,
|
||
|
|
"regularization/w1": 0.9061475992202759,
|
||
|
|
"rewards/chosen": 0.08294287706032777,
|
||
|
|
"rewards/margins": 0.19643550548350686,
|
||
|
|
"rewards/rejected": -0.11349262842317907,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 17.114286422729492,
|
||
|
|
"kl": 28.3470516204834,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -35719567.97650514,
|
||
|
|
"logits/rejected": -36538672.29382304,
|
||
|
|
"logps/chosen": -486.05892070484583,
|
||
|
|
"logps/rejected": -366.78078046744577,
|
||
|
|
"loss": 0.5914,
|
||
|
|
"loss/base_kto": 3.6441636085510254,
|
||
|
|
"metrics/advantage_var": 277.0114440917969,
|
||
|
|
"regularization/lipschitz_norm": 1102.2738037109375,
|
||
|
|
"regularization/mmd": 0.16112275421619415,
|
||
|
|
"regularization/rkhs_norm": 191.81280517578125,
|
||
|
|
"regularization/w1": 0.9259099960327148,
|
||
|
|
"rewards/chosen": 0.5076863965273953,
|
||
|
|
"rewards/margins": 0.30991189473322667,
|
||
|
|
"rewards/rejected": 0.19777450179416867,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 20.1845760345459,
|
||
|
|
"kl": 22.826278686523438,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -36458379.15689382,
|
||
|
|
"logits/rejected": -36623621.14374034,
|
||
|
|
"logps/chosen": -469.39659270998413,
|
||
|
|
"logps/rejected": -360.1577714451314,
|
||
|
|
"loss": 0.6075,
|
||
|
|
"loss/base_kto": 3.614739179611206,
|
||
|
|
"metrics/advantage_var": 394.32861328125,
|
||
|
|
"regularization/lipschitz_norm": 1266.2174072265625,
|
||
|
|
"regularization/mmd": 0.18139131367206573,
|
||
|
|
"regularization/rkhs_norm": 215.94204711914062,
|
||
|
|
"regularization/w1": 1.0636225938796997,
|
||
|
|
"rewards/chosen": 0.42696843698927794,
|
||
|
|
"rewards/margins": 0.3763576632748085,
|
||
|
|
"rewards/rejected": 0.050610773714469426,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 30.14720916748047,
|
||
|
|
"kl": 15.291643142700195,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -36967712.24883359,
|
||
|
|
"logits/rejected": -38327731.642072216,
|
||
|
|
"logps/chosen": -482.3858864696734,
|
||
|
|
"logps/rejected": -378.95947802197804,
|
||
|
|
"loss": 0.6284,
|
||
|
|
"loss/base_kto": 3.3046669960021973,
|
||
|
|
"metrics/advantage_var": 679.3814697265625,
|
||
|
|
"regularization/lipschitz_norm": 1763.589111328125,
|
||
|
|
"regularization/mmd": 0.24101614952087402,
|
||
|
|
"regularization/rkhs_norm": 286.9239807128906,
|
||
|
|
"regularization/w1": 1.481414794921875,
|
||
|
|
"rewards/chosen": 0.5338768847808126,
|
||
|
|
"rewards/margins": 0.7598774205870823,
|
||
|
|
"rewards/rejected": -0.22600053580626964,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 19.852436065673828,
|
||
|
|
"kl": 14.40466022491455,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -36692998.32098766,
|
||
|
|
"logits/rejected": -38535135.594936706,
|
||
|
|
"logps/chosen": -493.6412037037037,
|
||
|
|
"logps/rejected": -356.8260977056962,
|
||
|
|
"loss": 0.6061,
|
||
|
|
"loss/base_kto": 3.3381354808807373,
|
||
|
|
"metrics/advantage_var": 522.1862182617188,
|
||
|
|
"regularization/lipschitz_norm": 1542.626953125,
|
||
|
|
"regularization/mmd": 0.21481800079345703,
|
||
|
|
"regularization/rkhs_norm": 255.7357177734375,
|
||
|
|
"regularization/w1": 1.2958067655563354,
|
||
|
|
"rewards/chosen": 0.44550949850200133,
|
||
|
|
"rewards/margins": 0.7030229552087606,
|
||
|
|
"rewards/rejected": -0.2575134567067593,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 17.26534080505371,
|
||
|
|
"kl": 12.90103816986084,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -36583257.6,
|
||
|
|
"logits/rejected": -37187059.2,
|
||
|
|
"logps/chosen": -471.836767578125,
|
||
|
|
"logps/rejected": -396.18720703125,
|
||
|
|
"loss": 0.6197,
|
||
|
|
"loss/base_kto": 3.31327223777771,
|
||
|
|
"metrics/advantage_var": 666.0192260742188,
|
||
|
|
"regularization/lipschitz_norm": 1676.6455078125,
|
||
|
|
"regularization/mmd": 0.23618967831134796,
|
||
|
|
"regularization/rkhs_norm": 281.1781921386719,
|
||
|
|
"regularization/w1": 1.4083822965621948,
|
||
|
|
"rewards/chosen": 0.5340013980865479,
|
||
|
|
"rewards/margins": 0.7987452745437622,
|
||
|
|
"rewards/rejected": -0.26474387645721437,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 23.737882614135742,
|
||
|
|
"kl": 12.132939338684082,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -35278184.65408805,
|
||
|
|
"logits/rejected": -37071305.9378882,
|
||
|
|
"logps/chosen": -499.165634827044,
|
||
|
|
"logps/rejected": -377.6488014363354,
|
||
|
|
"loss": 0.6043,
|
||
|
|
"loss/base_kto": 3.3615880012512207,
|
||
|
|
"metrics/advantage_var": 558.8425903320312,
|
||
|
|
"regularization/lipschitz_norm": 1498.6766357421875,
|
||
|
|
"regularization/mmd": 0.2139439582824707,
|
||
|
|
"regularization/rkhs_norm": 254.69522094726562,
|
||
|
|
"regularization/w1": 1.2588883638381958,
|
||
|
|
"rewards/chosen": 0.44297598592890136,
|
||
|
|
"rewards/margins": 0.7101170082260675,
|
||
|
|
"rewards/rejected": -0.26714102229716613,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 15.236355781555176,
|
||
|
|
"kl": 17.843761444091797,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -35046467.43032159,
|
||
|
|
"logits/rejected": -36572099.57256778,
|
||
|
|
"logps/chosen": -445.42783307810106,
|
||
|
|
"logps/rejected": -381.4280801435407,
|
||
|
|
"loss": 0.5917,
|
||
|
|
"loss/base_kto": 3.2963523864746094,
|
||
|
|
"metrics/advantage_var": 551.6969604492188,
|
||
|
|
"regularization/lipschitz_norm": 1460.2900390625,
|
||
|
|
"regularization/mmd": 0.21053610742092133,
|
||
|
|
"regularization/rkhs_norm": 250.6382293701172,
|
||
|
|
"regularization/w1": 1.2266435623168945,
|
||
|
|
"rewards/chosen": 0.5309551057917783,
|
||
|
|
"rewards/margins": 0.7822974655141044,
|
||
|
|
"rewards/rejected": -0.25134235972232605,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 16.517234802246094,
|
||
|
|
"kl": 17.103546142578125,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -35180473.26844584,
|
||
|
|
"logits/rejected": -36107501.28771384,
|
||
|
|
"logps/chosen": -471.614010989011,
|
||
|
|
"logps/rejected": -369.73233378693624,
|
||
|
|
"loss": 0.6009,
|
||
|
|
"loss/base_kto": 3.380141019821167,
|
||
|
|
"metrics/advantage_var": 526.4321899414062,
|
||
|
|
"regularization/lipschitz_norm": 1447.9232177734375,
|
||
|
|
"regularization/mmd": 0.2106965035200119,
|
||
|
|
"regularization/rkhs_norm": 250.82920837402344,
|
||
|
|
"regularization/w1": 1.2162554264068604,
|
||
|
|
"rewards/chosen": 0.498858537179896,
|
||
|
|
"rewards/margins": 0.674133043507049,
|
||
|
|
"rewards/rejected": -0.175274506327153,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 13.176434516906738,
|
||
|
|
"kl": 15.887646675109863,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -33349247.42771982,
|
||
|
|
"logits/rejected": -35284178.18062397,
|
||
|
|
"logps/chosen": -476.1371087928465,
|
||
|
|
"logps/rejected": -368.3392857142857,
|
||
|
|
"loss": 0.6098,
|
||
|
|
"loss/base_kto": 3.3418033123016357,
|
||
|
|
"metrics/advantage_var": 591.3095092773438,
|
||
|
|
"regularization/lipschitz_norm": 1566.5013427734375,
|
||
|
|
"regularization/mmd": 0.2209499329328537,
|
||
|
|
"regularization/rkhs_norm": 263.0356750488281,
|
||
|
|
"regularization/w1": 1.3158612251281738,
|
||
|
|
"rewards/chosen": 0.5200721069881706,
|
||
|
|
"rewards/margins": 0.7304523229383708,
|
||
|
|
"rewards/rejected": -0.21038021595020012,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 18.833049774169922,
|
||
|
|
"kl": 20.219152450561523,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35351840.66468843,
|
||
|
|
"logits/rejected": -36238461.04290429,
|
||
|
|
"logps/chosen": -492.69621661721067,
|
||
|
|
"logps/rejected": -365.50549195544556,
|
||
|
|
"loss": 0.6237,
|
||
|
|
"loss/base_kto": 3.3600921630859375,
|
||
|
|
"metrics/advantage_var": 596.228759765625,
|
||
|
|
"regularization/lipschitz_norm": 1668.950439453125,
|
||
|
|
"regularization/mmd": 0.22760716080665588,
|
||
|
|
"regularization/rkhs_norm": 270.9609069824219,
|
||
|
|
"regularization/w1": 1.4019182920455933,
|
||
|
|
"rewards/chosen": 0.5726751219978672,
|
||
|
|
"rewards/margins": 0.7042602966591321,
|
||
|
|
"rewards/rejected": -0.13158517466126496,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 17.778493881225586,
|
||
|
|
"kl": 18.271499633789062,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -35268249.27388535,
|
||
|
|
"logits/rejected": -35581867.19018405,
|
||
|
|
"logps/chosen": -494.1021098726115,
|
||
|
|
"logps/rejected": -388.5136359279141,
|
||
|
|
"loss": 0.6139,
|
||
|
|
"loss/base_kto": 3.3075549602508545,
|
||
|
|
"metrics/advantage_var": 640.4003295898438,
|
||
|
|
"regularization/lipschitz_norm": 1638.5855712890625,
|
||
|
|
"regularization/mmd": 0.22699318826198578,
|
||
|
|
"regularization/rkhs_norm": 270.22998046875,
|
||
|
|
"regularization/w1": 1.3764116764068604,
|
||
|
|
"rewards/chosen": 0.5335830396907345,
|
||
|
|
"rewards/margins": 0.774583373137401,
|
||
|
|
"rewards/rejected": -0.24100033344666652,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 20.075347900390625,
|
||
|
|
"kl": 12.981180191040039,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -35015986.58682635,
|
||
|
|
"logits/rejected": -37582911.58169935,
|
||
|
|
"logps/chosen": -487.8549775449102,
|
||
|
|
"logps/rejected": -356.66048815359477,
|
||
|
|
"loss": 0.6053,
|
||
|
|
"loss/base_kto": 3.288564443588257,
|
||
|
|
"metrics/advantage_var": 623.2282104492188,
|
||
|
|
"regularization/lipschitz_norm": 1576.2935791015625,
|
||
|
|
"regularization/mmd": 0.22978658974170685,
|
||
|
|
"regularization/rkhs_norm": 273.55548095703125,
|
||
|
|
"regularization/w1": 1.3240865468978882,
|
||
|
|
"rewards/chosen": 0.5048243357035929,
|
||
|
|
"rewards/margins": 0.8054399330981497,
|
||
|
|
"rewards/rejected": -0.3006155973945568,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 16.932571411132812,
|
||
|
|
"kl": 31.370588302612305,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34778587.372907154,
|
||
|
|
"logits/rejected": -36209751.113964684,
|
||
|
|
"logps/chosen": -489.5570776255708,
|
||
|
|
"logps/rejected": -363.12103731942216,
|
||
|
|
"loss": 0.5876,
|
||
|
|
"loss/base_kto": 3.262110948562622,
|
||
|
|
"metrics/advantage_var": 507.5049743652344,
|
||
|
|
"regularization/lipschitz_norm": 1457.1304931640625,
|
||
|
|
"regularization/mmd": 0.21449874341487885,
|
||
|
|
"regularization/rkhs_norm": 255.3556671142578,
|
||
|
|
"regularization/w1": 1.2239896059036255,
|
||
|
|
"rewards/chosen": 0.7377876270125808,
|
||
|
|
"rewards/margins": 0.7224614929294131,
|
||
|
|
"rewards/rejected": 0.015326134083167698,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 19.6783504486084,
|
||
|
|
"kl": 25.338829040527344,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -35122894.568288855,
|
||
|
|
"logits/rejected": -34815159.141524106,
|
||
|
|
"logps/chosen": -472.63417386185245,
|
||
|
|
"logps/rejected": -354.4738287325039,
|
||
|
|
"loss": 0.589,
|
||
|
|
"loss/base_kto": 3.291515350341797,
|
||
|
|
"metrics/advantage_var": 518.9993286132812,
|
||
|
|
"regularization/lipschitz_norm": 1441.677978515625,
|
||
|
|
"regularization/mmd": 0.2093811333179474,
|
||
|
|
"regularization/rkhs_norm": 249.2632598876953,
|
||
|
|
"regularization/w1": 1.2110095024108887,
|
||
|
|
"rewards/chosen": 0.6328515452530171,
|
||
|
|
"rewards/margins": 0.7417727683254073,
|
||
|
|
"rewards/rejected": -0.10892122307239016,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 14.299549102783203,
|
||
|
|
"kl": 13.822909355163574,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -34826474.057142854,
|
||
|
|
"logits/rejected": -35530600.763076924,
|
||
|
|
"logps/chosen": -480.8300595238095,
|
||
|
|
"logps/rejected": -373.80310096153846,
|
||
|
|
"loss": 0.5948,
|
||
|
|
"loss/base_kto": 3.2905819416046143,
|
||
|
|
"metrics/advantage_var": 525.7487182617188,
|
||
|
|
"regularization/lipschitz_norm": 1492.8621826171875,
|
||
|
|
"regularization/mmd": 0.21371899545192719,
|
||
|
|
"regularization/rkhs_norm": 254.42739868164062,
|
||
|
|
"regularization/w1": 1.2540040016174316,
|
||
|
|
"rewards/chosen": 0.4819301060267857,
|
||
|
|
"rewards/margins": 0.7838381354363411,
|
||
|
|
"rewards/rejected": -0.3019080294095553,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 21.68499183654785,
|
||
|
|
"kl": 11.745469093322754,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -36805399.497584544,
|
||
|
|
"logits/rejected": -36395620.2245827,
|
||
|
|
"logps/chosen": -521.2514090177134,
|
||
|
|
"logps/rejected": -378.34476953717757,
|
||
|
|
"loss": 0.6245,
|
||
|
|
"loss/base_kto": 3.3373310565948486,
|
||
|
|
"metrics/advantage_var": 717.9779663085938,
|
||
|
|
"regularization/lipschitz_norm": 1694.2879638671875,
|
||
|
|
"regularization/mmd": 0.2353643923997879,
|
||
|
|
"regularization/rkhs_norm": 280.1957092285156,
|
||
|
|
"regularization/w1": 1.4232019186019897,
|
||
|
|
"rewards/chosen": 0.3695133473370194,
|
||
|
|
"rewards/margins": 0.7401284020334107,
|
||
|
|
"rewards/rejected": -0.37061505469639133,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 21.23410987854004,
|
||
|
|
"kl": 8.063403129577637,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34965694.433931485,
|
||
|
|
"logits/rejected": -36071813.94902549,
|
||
|
|
"logps/chosen": -462.24525897226755,
|
||
|
|
"logps/rejected": -379.5708395802099,
|
||
|
|
"loss": 0.6046,
|
||
|
|
"loss/base_kto": 3.344383955001831,
|
||
|
|
"metrics/advantage_var": 544.3243408203125,
|
||
|
|
"regularization/lipschitz_norm": 1522.578369140625,
|
||
|
|
"regularization/mmd": 0.21384473145008087,
|
||
|
|
"regularization/rkhs_norm": 254.57705688476562,
|
||
|
|
"regularization/w1": 1.2789658308029175,
|
||
|
|
"rewards/chosen": 0.28101740925774876,
|
||
|
|
"rewards/margins": 0.719642853615648,
|
||
|
|
"rewards/rejected": -0.4386254443578992,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 21.19676971435547,
|
||
|
|
"kl": 10.65280532836914,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34922690.43037975,
|
||
|
|
"logits/rejected": -36481046.12345679,
|
||
|
|
"logps/chosen": -491.1184731012658,
|
||
|
|
"logps/rejected": -386.28317901234567,
|
||
|
|
"loss": 0.6083,
|
||
|
|
"loss/base_kto": 3.267338514328003,
|
||
|
|
"metrics/advantage_var": 670.4760131835938,
|
||
|
|
"regularization/lipschitz_norm": 1626.25390625,
|
||
|
|
"regularization/mmd": 0.23278503119945526,
|
||
|
|
"regularization/rkhs_norm": 277.12506103515625,
|
||
|
|
"regularization/w1": 1.3660532236099243,
|
||
|
|
"rewards/chosen": 0.42179600196548656,
|
||
|
|
"rewards/margins": 0.803499508246833,
|
||
|
|
"rewards/rejected": -0.38170350628134647,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 17.94518280029297,
|
||
|
|
"kl": 7.550856113433838,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -34210965.97165354,
|
||
|
|
"logits/rejected": -35365899.1131783,
|
||
|
|
"logps/chosen": -454.8527559055118,
|
||
|
|
"logps/rejected": -370.1827277131783,
|
||
|
|
"loss": 0.6042,
|
||
|
|
"loss/base_kto": 3.37127685546875,
|
||
|
|
"metrics/advantage_var": 532.8030395507812,
|
||
|
|
"regularization/lipschitz_norm": 1485.170166015625,
|
||
|
|
"regularization/mmd": 0.2144092172384262,
|
||
|
|
"regularization/rkhs_norm": 255.2490692138672,
|
||
|
|
"regularization/w1": 1.2475429773330688,
|
||
|
|
"rewards/chosen": 0.2536710724117249,
|
||
|
|
"rewards/margins": 0.6749354369647772,
|
||
|
|
"rewards/rejected": -0.42126436455305233,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 21.139720916748047,
|
||
|
|
"kl": 12.362833976745605,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -33882437.02927581,
|
||
|
|
"logits/rejected": -34784679.55625991,
|
||
|
|
"logps/chosen": -472.1962153312789,
|
||
|
|
"logps/rejected": -360.87633716323296,
|
||
|
|
"loss": 0.6008,
|
||
|
|
"loss/base_kto": 3.286090850830078,
|
||
|
|
"metrics/advantage_var": 561.7142944335938,
|
||
|
|
"regularization/lipschitz_norm": 1551.6668701171875,
|
||
|
|
"regularization/mmd": 0.21712155640125275,
|
||
|
|
"regularization/rkhs_norm": 258.47802734375,
|
||
|
|
"regularization/w1": 1.3034000396728516,
|
||
|
|
"rewards/chosen": 0.4976035472240948,
|
||
|
|
"rewards/margins": 0.7835909780444692,
|
||
|
|
"rewards/rejected": -0.2859874308203744,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 16.639875411987305,
|
||
|
|
"kl": 9.712878227233887,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -35478420.123176664,
|
||
|
|
"logits/rejected": -36061906.823529415,
|
||
|
|
"logps/chosen": -482.0229943273906,
|
||
|
|
"logps/rejected": -369.93636877828055,
|
||
|
|
"loss": 0.5927,
|
||
|
|
"loss/base_kto": 3.3484184741973877,
|
||
|
|
"metrics/advantage_var": 488.9422912597656,
|
||
|
|
"regularization/lipschitz_norm": 1415.404541015625,
|
||
|
|
"regularization/mmd": 0.2046360969543457,
|
||
|
|
"regularization/rkhs_norm": 243.61441040039062,
|
||
|
|
"regularization/w1": 1.1889398097991943,
|
||
|
|
"rewards/chosen": 0.33435288588463585,
|
||
|
|
"rewards/margins": 0.6939580105178725,
|
||
|
|
"rewards/rejected": -0.3596051246332367,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 24.6983642578125,
|
||
|
|
"kl": 9.777175903320312,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34844887.245749615,
|
||
|
|
"logits/rejected": -35976772.751974724,
|
||
|
|
"logps/chosen": -485.0933636012365,
|
||
|
|
"logps/rejected": -366.4594687993681,
|
||
|
|
"loss": 0.6151,
|
||
|
|
"loss/base_kto": 3.35014271736145,
|
||
|
|
"metrics/advantage_var": 736.5438842773438,
|
||
|
|
"regularization/lipschitz_norm": 1605.5198974609375,
|
||
|
|
"regularization/mmd": 0.2221064418554306,
|
||
|
|
"regularization/rkhs_norm": 264.4124450683594,
|
||
|
|
"regularization/w1": 1.3486367464065552,
|
||
|
|
"rewards/chosen": 0.39864683077545404,
|
||
|
|
"rewards/margins": 0.6804745434747776,
|
||
|
|
"rewards/rejected": -0.28182771269932366,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 19.533191680908203,
|
||
|
|
"kl": 9.757631301879883,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -35173026.18633541,
|
||
|
|
"logits/rejected": -37269793.81132075,
|
||
|
|
"logps/chosen": -474.92294254658384,
|
||
|
|
"logps/rejected": -391.2468307783019,
|
||
|
|
"loss": 0.6054,
|
||
|
|
"loss/base_kto": 3.274069309234619,
|
||
|
|
"metrics/advantage_var": 546.0035400390625,
|
||
|
|
"regularization/lipschitz_norm": 1608.5828857421875,
|
||
|
|
"regularization/mmd": 0.21795068681240082,
|
||
|
|
"regularization/rkhs_norm": 259.465087890625,
|
||
|
|
"regularization/w1": 1.3512096405029297,
|
||
|
|
"rewards/chosen": 0.445318423442959,
|
||
|
|
"rewards/margins": 0.785911821435171,
|
||
|
|
"rewards/rejected": -0.34059339799221205,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 20.48908233642578,
|
||
|
|
"kl": 11.801558494567871,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -35335841.77198697,
|
||
|
|
"logits/rejected": -35890348.2042042,
|
||
|
|
"logps/chosen": -486.5277890879479,
|
||
|
|
"logps/rejected": -372.42149962462463,
|
||
|
|
"loss": 0.6001,
|
||
|
|
"loss/base_kto": 3.3054816722869873,
|
||
|
|
"metrics/advantage_var": 570.5050659179688,
|
||
|
|
"regularization/lipschitz_norm": 1523.2435302734375,
|
||
|
|
"regularization/mmd": 0.21592283248901367,
|
||
|
|
"regularization/rkhs_norm": 257.051025390625,
|
||
|
|
"regularization/w1": 1.2795246839523315,
|
||
|
|
"rewards/chosen": 0.4257883575141236,
|
||
|
|
"rewards/margins": 0.7725356556545492,
|
||
|
|
"rewards/rejected": -0.3467472981404256,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 19.19207191467285,
|
||
|
|
"kl": 13.553595542907715,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -34889640.18376723,
|
||
|
|
"logits/rejected": -36536169.74800638,
|
||
|
|
"logps/chosen": -473.4591787901991,
|
||
|
|
"logps/rejected": -378.58951355661884,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.3230807781219482,
|
||
|
|
"metrics/advantage_var": 545.9921875,
|
||
|
|
"regularization/lipschitz_norm": 1504.0831298828125,
|
||
|
|
"regularization/mmd": 0.21675610542297363,
|
||
|
|
"regularization/rkhs_norm": 258.04296875,
|
||
|
|
"regularization/w1": 1.2634297609329224,
|
||
|
|
"rewards/chosen": 0.4697913364100785,
|
||
|
|
"rewards/margins": 0.7035303831791674,
|
||
|
|
"rewards/rejected": -0.2337390467690889,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 15.541764259338379,
|
||
|
|
"kl": 12.717080116271973,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -35423642.24605678,
|
||
|
|
"logits/rejected": -37378539.39318886,
|
||
|
|
"logps/chosen": -505.2036179022082,
|
||
|
|
"logps/rejected": -369.9245839783282,
|
||
|
|
"loss": 0.6017,
|
||
|
|
"loss/base_kto": 3.327488660812378,
|
||
|
|
"metrics/advantage_var": 538.381591796875,
|
||
|
|
"regularization/lipschitz_norm": 1517.1551513671875,
|
||
|
|
"regularization/mmd": 0.2117990255355835,
|
||
|
|
"regularization/rkhs_norm": 252.14170837402344,
|
||
|
|
"regularization/w1": 1.2744102478027344,
|
||
|
|
"rewards/chosen": 0.42909361060109424,
|
||
|
|
"rewards/margins": 0.7177926396319089,
|
||
|
|
"rewards/rejected": -0.2886990290308146,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 18.025327682495117,
|
||
|
|
"kl": 12.646842956542969,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -35896735.90153846,
|
||
|
|
"logits/rejected": -35636058.20952381,
|
||
|
|
"logps/chosen": -471.96846153846155,
|
||
|
|
"logps/rejected": -390.17410714285717,
|
||
|
|
"loss": 0.604,
|
||
|
|
"loss/base_kto": 3.312110185623169,
|
||
|
|
"metrics/advantage_var": 576.8339233398438,
|
||
|
|
"regularization/lipschitz_norm": 1548.7454833984375,
|
||
|
|
"regularization/mmd": 0.2193342000246048,
|
||
|
|
"regularization/rkhs_norm": 261.1121520996094,
|
||
|
|
"regularization/w1": 1.3009463548660278,
|
||
|
|
"rewards/chosen": 0.4091322678786058,
|
||
|
|
"rewards/margins": 0.7262386783138737,
|
||
|
|
"rewards/rejected": -0.3171064104352679,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 17.990734100341797,
|
||
|
|
"kl": 14.885505676269531,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -35742065.609467454,
|
||
|
|
"logits/rejected": -35703214.62251656,
|
||
|
|
"logps/chosen": -480.8216068786982,
|
||
|
|
"logps/rejected": -376.9612737996689,
|
||
|
|
"loss": 0.6055,
|
||
|
|
"loss/base_kto": 3.234381914138794,
|
||
|
|
"metrics/advantage_var": 622.4717407226562,
|
||
|
|
"regularization/lipschitz_norm": 1642.2193603515625,
|
||
|
|
"regularization/mmd": 0.23044441640377045,
|
||
|
|
"regularization/rkhs_norm": 274.3385925292969,
|
||
|
|
"regularization/w1": 1.3794642686843872,
|
||
|
|
"rewards/chosen": 0.586639133430797,
|
||
|
|
"rewards/margins": 0.8144280712331433,
|
||
|
|
"rewards/rejected": -0.22778893780234633,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 19.09687042236328,
|
||
|
|
"kl": 13.606826782226562,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -36210587.771615006,
|
||
|
|
"logits/rejected": -36060106.266866565,
|
||
|
|
"logps/chosen": -481.3544045676998,
|
||
|
|
"logps/rejected": -383.96675880809596,
|
||
|
|
"loss": 0.6081,
|
||
|
|
"loss/base_kto": 3.22845721244812,
|
||
|
|
"metrics/advantage_var": 648.509521484375,
|
||
|
|
"regularization/lipschitz_norm": 1670.5927734375,
|
||
|
|
"regularization/mmd": 0.23286108672618866,
|
||
|
|
"regularization/rkhs_norm": 277.215576171875,
|
||
|
|
"regularization/w1": 1.403298020362854,
|
||
|
|
"rewards/chosen": 0.5233538629180261,
|
||
|
|
"rewards/margins": 0.8296589089423771,
|
||
|
|
"rewards/rejected": -0.3063050460243511,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 21.471250534057617,
|
||
|
|
"kl": 9.653098106384277,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -34026022.88607595,
|
||
|
|
"logits/rejected": -35710957.03703704,
|
||
|
|
"logps/chosen": -485.3389042721519,
|
||
|
|
"logps/rejected": -362.10927854938274,
|
||
|
|
"loss": 0.6016,
|
||
|
|
"loss/base_kto": 3.334829092025757,
|
||
|
|
"metrics/advantage_var": 608.798828125,
|
||
|
|
"regularization/lipschitz_norm": 1510.160888671875,
|
||
|
|
"regularization/mmd": 0.20944009721279144,
|
||
|
|
"regularization/rkhs_norm": 249.3334503173828,
|
||
|
|
"regularization/w1": 1.2685352563858032,
|
||
|
|
"rewards/chosen": 0.2437453209599362,
|
||
|
|
"rewards/margins": 0.6843596209695813,
|
||
|
|
"rewards/rejected": -0.4406143000096451,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 22.113142013549805,
|
||
|
|
"kl": 11.807127952575684,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -35004644.9689441,
|
||
|
|
"logits/rejected": -35935972.628930815,
|
||
|
|
"logps/chosen": -472.87990100931677,
|
||
|
|
"logps/rejected": -383.3904530267296,
|
||
|
|
"loss": 0.6044,
|
||
|
|
"loss/base_kto": 3.2573845386505127,
|
||
|
|
"metrics/advantage_var": 629.4389038085938,
|
||
|
|
"regularization/lipschitz_norm": 1610.2572021484375,
|
||
|
|
"regularization/mmd": 0.22549353539943695,
|
||
|
|
"regularization/rkhs_norm": 268.4446716308594,
|
||
|
|
"regularization/w1": 1.3526160717010498,
|
||
|
|
"rewards/chosen": 0.4835328285738548,
|
||
|
|
"rewards/margins": 0.8426700457776308,
|
||
|
|
"rewards/rejected": -0.35913721720377606,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 17.822072982788086,
|
||
|
|
"kl": 12.865683555603027,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -34269693.625966,
|
||
|
|
"logits/rejected": -35976850.401263826,
|
||
|
|
"logps/chosen": -500.7642967542504,
|
||
|
|
"logps/rejected": -386.6487954186414,
|
||
|
|
"loss": 0.6125,
|
||
|
|
"loss/base_kto": 3.3427696228027344,
|
||
|
|
"metrics/advantage_var": 572.8134155273438,
|
||
|
|
"regularization/lipschitz_norm": 1589.7178955078125,
|
||
|
|
"regularization/mmd": 0.22191600501537323,
|
||
|
|
"regularization/rkhs_norm": 264.1857604980469,
|
||
|
|
"regularization/w1": 1.3353630304336548,
|
||
|
|
"rewards/chosen": 0.4387526829054772,
|
||
|
|
"rewards/margins": 0.7057367516690534,
|
||
|
|
"rewards/rejected": -0.2669840687635762,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 12.618535041809082,
|
||
|
|
"kl": 12.74691104888916,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -34078279.49919743,
|
||
|
|
"logits/rejected": -34741355.543378994,
|
||
|
|
"logps/chosen": -492.42676565008026,
|
||
|
|
"logps/rejected": -354.0341990106545,
|
||
|
|
"loss": 0.5933,
|
||
|
|
"loss/base_kto": 3.249702215194702,
|
||
|
|
"metrics/advantage_var": 520.4142456054688,
|
||
|
|
"regularization/lipschitz_norm": 1528.2381591796875,
|
||
|
|
"regularization/mmd": 0.21315963566303253,
|
||
|
|
"regularization/rkhs_norm": 253.761474609375,
|
||
|
|
"regularization/w1": 1.2837201356887817,
|
||
|
|
"rewards/chosen": 0.511796537984049,
|
||
|
|
"rewards/margins": 0.7991691677390313,
|
||
|
|
"rewards/rejected": -0.2873726297549824,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 20.017213821411133,
|
||
|
|
"kl": 7.575507640838623,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -34369425.6481775,
|
||
|
|
"logits/rejected": -35838734.59476117,
|
||
|
|
"logps/chosen": -482.71711568938196,
|
||
|
|
"logps/rejected": -385.2330026964561,
|
||
|
|
"loss": 0.6088,
|
||
|
|
"loss/base_kto": 3.3300721645355225,
|
||
|
|
"metrics/advantage_var": 551.4451904296875,
|
||
|
|
"regularization/lipschitz_norm": 1572.4085693359375,
|
||
|
|
"regularization/mmd": 0.21954384446144104,
|
||
|
|
"regularization/rkhs_norm": 261.3617248535156,
|
||
|
|
"regularization/w1": 1.320823311805725,
|
||
|
|
"rewards/chosen": 0.377362476475832,
|
||
|
|
"rewards/margins": 0.7479974753322457,
|
||
|
|
"rewards/rejected": -0.3706349988564137,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 13.82175350189209,
|
||
|
|
"kl": 8.901938438415527,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -33453443.062200956,
|
||
|
|
"logits/rejected": -34858838.64012251,
|
||
|
|
"logps/chosen": -480.2731259968102,
|
||
|
|
"logps/rejected": -383.6234207503828,
|
||
|
|
"loss": 0.5934,
|
||
|
|
"loss/base_kto": 3.3000335693359375,
|
||
|
|
"metrics/advantage_var": 538.3826293945312,
|
||
|
|
"regularization/lipschitz_norm": 1468.3470458984375,
|
||
|
|
"regularization/mmd": 0.21402302384376526,
|
||
|
|
"regularization/rkhs_norm": 254.78929138183594,
|
||
|
|
"regularization/w1": 1.2334115505218506,
|
||
|
|
"rewards/chosen": 0.40801843767911433,
|
||
|
|
"rewards/margins": 0.7755517256738262,
|
||
|
|
"rewards/rejected": -0.3675332879947119,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 16.749231338500977,
|
||
|
|
"kl": 6.636883735656738,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34588749.6492891,
|
||
|
|
"logits/rejected": -34832643.561051,
|
||
|
|
"logps/chosen": -456.658570300158,
|
||
|
|
"logps/rejected": -403.6516132148377,
|
||
|
|
"loss": 0.6128,
|
||
|
|
"loss/base_kto": 3.294590711593628,
|
||
|
|
"metrics/advantage_var": 709.7509155273438,
|
||
|
|
"regularization/lipschitz_norm": 1638.9635009765625,
|
||
|
|
"regularization/mmd": 0.23090210556983948,
|
||
|
|
"regularization/rkhs_norm": 274.8834533691406,
|
||
|
|
"regularization/w1": 1.3767293691635132,
|
||
|
|
"rewards/chosen": 0.3069215265307193,
|
||
|
|
"rewards/margins": 0.7958374666872108,
|
||
|
|
"rewards/rejected": -0.4889159401564915,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 21.665218353271484,
|
||
|
|
"kl": 8.190171241760254,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -34391653.085072234,
|
||
|
|
"logits/rejected": -34643344.56012177,
|
||
|
|
"logps/chosen": -498.69166332263245,
|
||
|
|
"logps/rejected": -374.8472460045662,
|
||
|
|
"loss": 0.605,
|
||
|
|
"loss/base_kto": 3.238154649734497,
|
||
|
|
"metrics/advantage_var": 579.7916259765625,
|
||
|
|
"regularization/lipschitz_norm": 1639.0191650390625,
|
||
|
|
"regularization/mmd": 0.22491280734539032,
|
||
|
|
"regularization/rkhs_norm": 267.7533264160156,
|
||
|
|
"regularization/w1": 1.3767759799957275,
|
||
|
|
"rewards/chosen": 0.34365690424201195,
|
||
|
|
"rewards/margins": 0.8382388961025047,
|
||
|
|
"rewards/rejected": -0.4945819918604928,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 12.00883960723877,
|
||
|
|
"kl": 8.099600791931152,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -34364068.256651014,
|
||
|
|
"logits/rejected": -35125107.41965678,
|
||
|
|
"logps/chosen": -481.04171557120503,
|
||
|
|
"logps/rejected": -391.90459243369736,
|
||
|
|
"loss": 0.5997,
|
||
|
|
"loss/base_kto": 3.3013312816619873,
|
||
|
|
"metrics/advantage_var": 561.3448486328125,
|
||
|
|
"regularization/lipschitz_norm": 1522.6522216796875,
|
||
|
|
"regularization/mmd": 0.21698199212551117,
|
||
|
|
"regularization/rkhs_norm": 258.3118591308594,
|
||
|
|
"regularization/w1": 1.2790278196334839,
|
||
|
|
"rewards/chosen": 0.3362527289114461,
|
||
|
|
"rewards/margins": 0.7826477697716743,
|
||
|
|
"rewards/rejected": -0.44639504086022813,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 16.841245651245117,
|
||
|
|
"kl": 9.618056297302246,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -35000446.41975309,
|
||
|
|
"logits/rejected": -36844343.088607594,
|
||
|
|
"logps/chosen": -492.89515817901236,
|
||
|
|
"logps/rejected": -367.14542128164555,
|
||
|
|
"loss": 0.6085,
|
||
|
|
"loss/base_kto": 3.3525917530059814,
|
||
|
|
"metrics/advantage_var": 560.1251831054688,
|
||
|
|
"regularization/lipschitz_norm": 1544.7421875,
|
||
|
|
"regularization/mmd": 0.21756787598133087,
|
||
|
|
"regularization/rkhs_norm": 259.0093688964844,
|
||
|
|
"regularization/w1": 1.2975834608078003,
|
||
|
|
"rewards/chosen": 0.34516786645959924,
|
||
|
|
"rewards/margins": 0.7210477094833581,
|
||
|
|
"rewards/rejected": -0.3758798430237589,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 21.1080379486084,
|
||
|
|
"kl": 11.35669231414795,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -33410666.9010989,
|
||
|
|
"logits/rejected": -34769791.00466563,
|
||
|
|
"logps/chosen": -498.6289246467818,
|
||
|
|
"logps/rejected": -381.9893565318818,
|
||
|
|
"loss": 0.6024,
|
||
|
|
"loss/base_kto": 3.252659559249878,
|
||
|
|
"metrics/advantage_var": 662.2719116210938,
|
||
|
|
"regularization/lipschitz_norm": 1594.5869140625,
|
||
|
|
"regularization/mmd": 0.2272685021162033,
|
||
|
|
"regularization/rkhs_norm": 270.5577697753906,
|
||
|
|
"regularization/w1": 1.339453101158142,
|
||
|
|
"rewards/chosen": 0.5052544370738079,
|
||
|
|
"rewards/margins": 0.8519850267550506,
|
||
|
|
"rewards/rejected": -0.3467305896812427,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 15.84586238861084,
|
||
|
|
"kl": 10.274452209472656,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -33943520.87537994,
|
||
|
|
"logits/rejected": -35674970.838709675,
|
||
|
|
"logps/chosen": -489.6505509118541,
|
||
|
|
"logps/rejected": -350.32694052419356,
|
||
|
|
"loss": 0.563,
|
||
|
|
"loss/base_kto": 3.2420356273651123,
|
||
|
|
"metrics/advantage_var": 389.9266052246094,
|
||
|
|
"regularization/lipschitz_norm": 1279.0777587890625,
|
||
|
|
"regularization/mmd": 0.18764524161815643,
|
||
|
|
"regularization/rkhs_norm": 223.3871612548828,
|
||
|
|
"regularization/w1": 1.074425220489502,
|
||
|
|
"rewards/chosen": 0.4061503308884641,
|
||
|
|
"rewards/margins": 0.8001105576242251,
|
||
|
|
"rewards/rejected": -0.3939602267357611,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 14.673681259155273,
|
||
|
|
"kl": 13.122614860534668,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -33392299.20760697,
|
||
|
|
"logits/rejected": -34548346.280431435,
|
||
|
|
"logps/chosen": -464.9544374009509,
|
||
|
|
"logps/rejected": -392.4648979198767,
|
||
|
|
"loss": 0.5569,
|
||
|
|
"loss/base_kto": 3.1842801570892334,
|
||
|
|
"metrics/advantage_var": 402.1956481933594,
|
||
|
|
"regularization/lipschitz_norm": 1288.0323486328125,
|
||
|
|
"regularization/mmd": 0.1890570968389511,
|
||
|
|
"regularization/rkhs_norm": 225.0679931640625,
|
||
|
|
"regularization/w1": 1.0819472074508667,
|
||
|
|
"rewards/chosen": 0.5139663140104992,
|
||
|
|
"rewards/margins": 0.839497931161886,
|
||
|
|
"rewards/rejected": -0.32553161715138673,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 15.556137084960938,
|
||
|
|
"kl": 10.652277946472168,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -34607248.77761414,
|
||
|
|
"logits/rejected": -35966676.12645591,
|
||
|
|
"logps/chosen": -488.3823177466863,
|
||
|
|
"logps/rejected": -364.26169925124793,
|
||
|
|
"loss": 0.5582,
|
||
|
|
"loss/base_kto": 3.1443049907684326,
|
||
|
|
"metrics/advantage_var": 442.4376525878906,
|
||
|
|
"regularization/lipschitz_norm": 1332.1800537109375,
|
||
|
|
"regularization/mmd": 0.20206427574157715,
|
||
|
|
"regularization/rkhs_norm": 240.5526885986328,
|
||
|
|
"regularization/w1": 1.119031310081482,
|
||
|
|
"rewards/chosen": 0.5578369679963642,
|
||
|
|
"rewards/margins": 0.9110108316659643,
|
||
|
|
"rewards/rejected": -0.35317386366960013,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 18.974214553833008,
|
||
|
|
"kl": 14.737631797790527,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -35975270.702064894,
|
||
|
|
"logits/rejected": -36277033.674418606,
|
||
|
|
"logps/chosen": -497.10550331858406,
|
||
|
|
"logps/rejected": -396.29347487541526,
|
||
|
|
"loss": 0.5696,
|
||
|
|
"loss/base_kto": 3.211864471435547,
|
||
|
|
"metrics/advantage_var": 431.98687744140625,
|
||
|
|
"regularization/lipschitz_norm": 1365.83544921875,
|
||
|
|
"regularization/mmd": 0.19773311913013458,
|
||
|
|
"regularization/rkhs_norm": 235.39659118652344,
|
||
|
|
"regularization/w1": 1.1473017930984497,
|
||
|
|
"rewards/chosen": 0.5352629264899059,
|
||
|
|
"rewards/margins": 0.8344073291808061,
|
||
|
|
"rewards/rejected": -0.29914440269090015,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 17.793354034423828,
|
||
|
|
"kl": 15.390420913696289,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -34687612.121212125,
|
||
|
|
"logits/rejected": -36155078.19354839,
|
||
|
|
"logps/chosen": -484.9957386363636,
|
||
|
|
"logps/rejected": -385.4767137096774,
|
||
|
|
"loss": 0.5633,
|
||
|
|
"loss/base_kto": 3.226102828979492,
|
||
|
|
"metrics/advantage_var": 418.45245361328125,
|
||
|
|
"regularization/lipschitz_norm": 1294.6143798828125,
|
||
|
|
"regularization/mmd": 0.19300396740436554,
|
||
|
|
"regularization/rkhs_norm": 229.7666015625,
|
||
|
|
"regularization/w1": 1.087476134300232,
|
||
|
|
"rewards/chosen": 0.5520561911843039,
|
||
|
|
"rewards/margins": 0.7991140505435529,
|
||
|
|
"rewards/rejected": -0.24705785935924898,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 18.184839248657227,
|
||
|
|
"kl": 17.659698486328125,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -34934539.32743363,
|
||
|
|
"logits/rejected": -35940869.10299003,
|
||
|
|
"logps/chosen": -495.3005162241888,
|
||
|
|
"logps/rejected": -388.21631021594686,
|
||
|
|
"loss": 0.5557,
|
||
|
|
"loss/base_kto": 3.187596559524536,
|
||
|
|
"metrics/advantage_var": 409.62841796875,
|
||
|
|
"regularization/lipschitz_norm": 1267.803466796875,
|
||
|
|
"regularization/mmd": 0.19272783398628235,
|
||
|
|
"regularization/rkhs_norm": 229.43789672851562,
|
||
|
|
"regularization/w1": 1.0649548768997192,
|
||
|
|
"rewards/chosen": 0.6390756252592644,
|
||
|
|
"rewards/margins": 0.835071146171004,
|
||
|
|
"rewards/rejected": -0.19599552091173952,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 15.002678871154785,
|
||
|
|
"kl": 22.223430633544922,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -34828452.36597111,
|
||
|
|
"logits/rejected": -35678684.93150685,
|
||
|
|
"logps/chosen": -474.8338683788122,
|
||
|
|
"logps/rejected": -376.26436453576866,
|
||
|
|
"loss": 0.5523,
|
||
|
|
"loss/base_kto": 3.159904956817627,
|
||
|
|
"metrics/advantage_var": 433.2723083496094,
|
||
|
|
"regularization/lipschitz_norm": 1267.8875732421875,
|
||
|
|
"regularization/mmd": 0.19317255914211273,
|
||
|
|
"regularization/rkhs_norm": 229.96731567382812,
|
||
|
|
"regularization/w1": 1.0650255680084229,
|
||
|
|
"rewards/chosen": 0.6723762622423004,
|
||
|
|
"rewards/margins": 0.8744663586797052,
|
||
|
|
"rewards/rejected": -0.20209009643740486,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 12.862616539001465,
|
||
|
|
"kl": 14.726593017578125,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34499999.65163297,
|
||
|
|
"logits/rejected": -34946869.45054945,
|
||
|
|
"logps/chosen": -487.72657465007774,
|
||
|
|
"logps/rejected": -349.1521291208791,
|
||
|
|
"loss": 0.5632,
|
||
|
|
"loss/base_kto": 3.1749675273895264,
|
||
|
|
"metrics/advantage_var": 461.6251525878906,
|
||
|
|
"regularization/lipschitz_norm": 1347.8260498046875,
|
||
|
|
"regularization/mmd": 0.19879385828971863,
|
||
|
|
"regularization/rkhs_norm": 236.6593780517578,
|
||
|
|
"regularization/w1": 1.1321738958358765,
|
||
|
|
"rewards/chosen": 0.5931191926425204,
|
||
|
|
"rewards/margins": 0.8781586901497564,
|
||
|
|
"rewards/rejected": -0.28503949750723606,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 13.166388511657715,
|
||
|
|
"kl": 17.485754013061523,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -34458167.439490445,
|
||
|
|
"logits/rejected": -35242247.85276074,
|
||
|
|
"logps/chosen": -442.3670382165605,
|
||
|
|
"logps/rejected": -391.0656154141104,
|
||
|
|
"loss": 0.5546,
|
||
|
|
"loss/base_kto": 3.207674503326416,
|
||
|
|
"metrics/advantage_var": 432.341064453125,
|
||
|
|
"regularization/lipschitz_norm": 1239.1695556640625,
|
||
|
|
"regularization/mmd": 0.18834565579891205,
|
||
|
|
"regularization/rkhs_norm": 224.2209930419922,
|
||
|
|
"regularization/w1": 1.0409023761749268,
|
||
|
|
"rewards/chosen": 0.5516619348222283,
|
||
|
|
"rewards/margins": 0.8118083274572695,
|
||
|
|
"rewards/rejected": -0.26014639263504125,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 15.19034194946289,
|
||
|
|
"kl": 10.276593208312988,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -34619947.60883281,
|
||
|
|
"logits/rejected": -35284015.554179564,
|
||
|
|
"logps/chosen": -468.8888505520505,
|
||
|
|
"logps/rejected": -375.39357585139317,
|
||
|
|
"loss": 0.5575,
|
||
|
|
"loss/base_kto": 3.203594923019409,
|
||
|
|
"metrics/advantage_var": 395.5413513183594,
|
||
|
|
"regularization/lipschitz_norm": 1271.7320556640625,
|
||
|
|
"regularization/mmd": 0.18793752789497375,
|
||
|
|
"regularization/rkhs_norm": 223.7351531982422,
|
||
|
|
"regularization/w1": 1.0682549476623535,
|
||
|
|
"rewards/chosen": 0.45097327006728116,
|
||
|
|
"rewards/margins": 0.8379109340874533,
|
||
|
|
"rewards/rejected": -0.3869376640201722,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 13.577218055725098,
|
||
|
|
"kl": 11.351633071899414,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -34957907.96141479,
|
||
|
|
"logits/rejected": -36514862.68693009,
|
||
|
|
"logps/chosen": -495.5143689710611,
|
||
|
|
"logps/rejected": -358.3359612462006,
|
||
|
|
"loss": 0.5599,
|
||
|
|
"loss/base_kto": 3.219320774078369,
|
||
|
|
"metrics/advantage_var": 407.5980224609375,
|
||
|
|
"regularization/lipschitz_norm": 1273.7225341796875,
|
||
|
|
"regularization/mmd": 0.18991895020008087,
|
||
|
|
"regularization/rkhs_norm": 226.09402465820312,
|
||
|
|
"regularization/w1": 1.0699270963668823,
|
||
|
|
"rewards/chosen": 0.4134593117275422,
|
||
|
|
"rewards/margins": 0.8221395869460073,
|
||
|
|
"rewards/rejected": -0.40868027521846506,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 17.006799697875977,
|
||
|
|
"kl": 13.061903953552246,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35450206.77112135,
|
||
|
|
"logits/rejected": -36295626.27662957,
|
||
|
|
"logps/chosen": -459.8193644393241,
|
||
|
|
"logps/rejected": -370.8439487281399,
|
||
|
|
"loss": 0.55,
|
||
|
|
"loss/base_kto": 3.189333438873291,
|
||
|
|
"metrics/advantage_var": 382.8031311035156,
|
||
|
|
"regularization/lipschitz_norm": 1222.7635498046875,
|
||
|
|
"regularization/mmd": 0.18362049758434296,
|
||
|
|
"regularization/rkhs_norm": 218.59580993652344,
|
||
|
|
"regularization/w1": 1.0271214246749878,
|
||
|
|
"rewards/chosen": 0.490606211297523,
|
||
|
|
"rewards/margins": 0.8540748120573476,
|
||
|
|
"rewards/rejected": -0.3634686007598246,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 17.623613357543945,
|
||
|
|
"kl": 14.34494686126709,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -34216982.47021943,
|
||
|
|
"logits/rejected": -34302519.82554517,
|
||
|
|
"logps/chosen": -480.9950039184953,
|
||
|
|
"logps/rejected": -356.5640332943925,
|
||
|
|
"loss": 0.5585,
|
||
|
|
"loss/base_kto": 3.2039711475372314,
|
||
|
|
"metrics/advantage_var": 440.133544921875,
|
||
|
|
"regularization/lipschitz_norm": 1274.4517822265625,
|
||
|
|
"regularization/mmd": 0.19310761988162994,
|
||
|
|
"regularization/rkhs_norm": 229.8900146484375,
|
||
|
|
"regularization/w1": 1.0705394744873047,
|
||
|
|
"rewards/chosen": 0.5658861118424275,
|
||
|
|
"rewards/margins": 0.8475557523092305,
|
||
|
|
"rewards/rejected": -0.281669640466803,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 18.474836349487305,
|
||
|
|
"kl": 11.145959854125977,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -33727795.67334361,
|
||
|
|
"logits/rejected": -36828096.70998415,
|
||
|
|
"logps/chosen": -513.0128563174114,
|
||
|
|
"logps/rejected": -379.7720384310618,
|
||
|
|
"loss": 0.5523,
|
||
|
|
"loss/base_kto": 3.1854116916656494,
|
||
|
|
"metrics/advantage_var": 385.63336181640625,
|
||
|
|
"regularization/lipschitz_norm": 1243.1856689453125,
|
||
|
|
"regularization/mmd": 0.18834401667118073,
|
||
|
|
"regularization/rkhs_norm": 224.21910095214844,
|
||
|
|
"regularization/w1": 1.0442759990692139,
|
||
|
|
"rewards/chosen": 0.539585765962057,
|
||
|
|
"rewards/margins": 0.837411381266076,
|
||
|
|
"rewards/rejected": -0.2978256153040189,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 16.720338821411133,
|
||
|
|
"kl": 11.830164909362793,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -34697306.55782313,
|
||
|
|
"logits/rejected": -35999335.58381503,
|
||
|
|
"logps/chosen": -488.01615646258506,
|
||
|
|
"logps/rejected": -391.93781611271675,
|
||
|
|
"loss": 0.5543,
|
||
|
|
"loss/base_kto": 3.1345040798187256,
|
||
|
|
"metrics/advantage_var": 429.4234924316406,
|
||
|
|
"regularization/lipschitz_norm": 1317.3709716796875,
|
||
|
|
"regularization/mmd": 0.19330470263957977,
|
||
|
|
"regularization/rkhs_norm": 230.12466430664062,
|
||
|
|
"regularization/w1": 1.1065915822982788,
|
||
|
|
"rewards/chosen": 0.4993897522387861,
|
||
|
|
"rewards/margins": 0.8876647223209755,
|
||
|
|
"rewards/rejected": -0.3882749700821893,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 17.21449851989746,
|
||
|
|
"kl": 8.917579650878906,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -34767588.5568,
|
||
|
|
"logits/rejected": -35621611.28549618,
|
||
|
|
"logps/chosen": -501.3483,
|
||
|
|
"logps/rejected": -380.98365935114504,
|
||
|
|
"loss": 0.5667,
|
||
|
|
"loss/base_kto": 3.2373528480529785,
|
||
|
|
"metrics/advantage_var": 441.436767578125,
|
||
|
|
"regularization/lipschitz_norm": 1307.6292724609375,
|
||
|
|
"regularization/mmd": 0.1978851854801178,
|
||
|
|
"regularization/rkhs_norm": 235.5775909423828,
|
||
|
|
"regularization/w1": 1.098408579826355,
|
||
|
|
"rewards/chosen": 0.37561064453125,
|
||
|
|
"rewards/margins": 0.8065592426795085,
|
||
|
|
"rewards/rejected": -0.4309485981482586,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 13.06633472442627,
|
||
|
|
"kl": 8.588080406188965,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -34330590.69918699,
|
||
|
|
"logits/rejected": -35925837.37744361,
|
||
|
|
"logps/chosen": -493.2400406504065,
|
||
|
|
"logps/rejected": -349.3700187969925,
|
||
|
|
"loss": 0.5537,
|
||
|
|
"loss/base_kto": 3.1561896800994873,
|
||
|
|
"metrics/advantage_var": 422.1860046386719,
|
||
|
|
"regularization/lipschitz_norm": 1285.82568359375,
|
||
|
|
"regularization/mmd": 0.19351409375667572,
|
||
|
|
"regularization/rkhs_norm": 230.37393188476562,
|
||
|
|
"regularization/w1": 1.0800936222076416,
|
||
|
|
"rewards/chosen": 0.4347711826727642,
|
||
|
|
"rewards/margins": 0.886212346617783,
|
||
|
|
"rewards/rejected": -0.4514411639450188,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 13.171441078186035,
|
||
|
|
"kl": 8.947189331054688,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -34306215.24807396,
|
||
|
|
"logits/rejected": -36405744.58320127,
|
||
|
|
"logps/chosen": -476.3931047765794,
|
||
|
|
"logps/rejected": -364.52929377971475,
|
||
|
|
"loss": 0.553,
|
||
|
|
"loss/base_kto": 3.2210700511932373,
|
||
|
|
"metrics/advantage_var": 362.1111145019531,
|
||
|
|
"regularization/lipschitz_norm": 1216.04833984375,
|
||
|
|
"regularization/mmd": 0.18131333589553833,
|
||
|
|
"regularization/rkhs_norm": 215.84922790527344,
|
||
|
|
"regularization/w1": 1.021480679512024,
|
||
|
|
"rewards/chosen": 0.4055867481672525,
|
||
|
|
"rewards/margins": 0.8161228477054359,
|
||
|
|
"rewards/rejected": -0.41053609953818343,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 14.532591819763184,
|
||
|
|
"kl": 8.963698387145996,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -33834569.571847506,
|
||
|
|
"logits/rejected": -34866480.80267558,
|
||
|
|
"logps/chosen": -479.7925219941349,
|
||
|
|
"logps/rejected": -356.2524561036789,
|
||
|
|
"loss": 0.553,
|
||
|
|
"loss/base_kto": 3.202587127685547,
|
||
|
|
"metrics/advantage_var": 395.7680358886719,
|
||
|
|
"regularization/lipschitz_norm": 1235.2418212890625,
|
||
|
|
"regularization/mmd": 0.18388526141643524,
|
||
|
|
"regularization/rkhs_norm": 218.9110107421875,
|
||
|
|
"regularization/w1": 1.0376030206680298,
|
||
|
|
"rewards/chosen": 0.4610707571080004,
|
||
|
|
"rewards/margins": 0.85213384765886,
|
||
|
|
"rewards/rejected": -0.39106309055085964,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 16.621479034423828,
|
||
|
|
"kl": 13.260581016540527,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -33507639.51807229,
|
||
|
|
"logits/rejected": -34799286.85714286,
|
||
|
|
"logps/chosen": -488.0190135542169,
|
||
|
|
"logps/rejected": -379.2658532873377,
|
||
|
|
"loss": 0.5604,
|
||
|
|
"loss/base_kto": 3.1591362953186035,
|
||
|
|
"metrics/advantage_var": 432.0214538574219,
|
||
|
|
"regularization/lipschitz_norm": 1340.5941162109375,
|
||
|
|
"regularization/mmd": 0.19805912673473358,
|
||
|
|
"regularization/rkhs_norm": 235.78466796875,
|
||
|
|
"regularization/w1": 1.1260989904403687,
|
||
|
|
"rewards/chosen": 0.5622578816241529,
|
||
|
|
"rewards/margins": 0.8873437569000001,
|
||
|
|
"rewards/rejected": -0.3250858752758472,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 15.701424598693848,
|
||
|
|
"kl": 14.250332832336426,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35499924.21052632,
|
||
|
|
"logits/rejected": -35915858.28571428,
|
||
|
|
"logps/chosen": -473.46273643092104,
|
||
|
|
"logps/rejected": -372.7054501488095,
|
||
|
|
"loss": 0.5429,
|
||
|
|
"loss/base_kto": 3.1128509044647217,
|
||
|
|
"metrics/advantage_var": 417.1114196777344,
|
||
|
|
"regularization/lipschitz_norm": 1239.255615234375,
|
||
|
|
"regularization/mmd": 0.1897449940443039,
|
||
|
|
"regularization/rkhs_norm": 225.8868865966797,
|
||
|
|
"regularization/w1": 1.0409746170043945,
|
||
|
|
"rewards/chosen": 0.5564124960648386,
|
||
|
|
"rewards/margins": 0.9140466090133017,
|
||
|
|
"rewards/rejected": -0.3576341129484631,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 15.832449913024902,
|
||
|
|
"kl": 14.343947410583496,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -34636168.50695518,
|
||
|
|
"logits/rejected": -35802016.55608215,
|
||
|
|
"logps/chosen": -473.04337326120555,
|
||
|
|
"logps/rejected": -390.03134873617694,
|
||
|
|
"loss": 0.5578,
|
||
|
|
"loss/base_kto": 3.1775963306427,
|
||
|
|
"metrics/advantage_var": 383.69873046875,
|
||
|
|
"regularization/lipschitz_norm": 1305.6473388671875,
|
||
|
|
"regularization/mmd": 0.18771111965179443,
|
||
|
|
"regularization/rkhs_norm": 223.4656219482422,
|
||
|
|
"regularization/w1": 1.0967435836791992,
|
||
|
|
"rewards/chosen": 0.5248484781020576,
|
||
|
|
"rewards/margins": 0.8445120577204422,
|
||
|
|
"rewards/rejected": -0.31966357961838465,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 16.625328063964844,
|
||
|
|
"kl": 14.722457885742188,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -34853143.27272727,
|
||
|
|
"logits/rejected": -35681442.69158878,
|
||
|
|
"logps/chosen": -484.8646159874608,
|
||
|
|
"logps/rejected": -366.8214563862928,
|
||
|
|
"loss": 0.5522,
|
||
|
|
"loss/base_kto": 3.22148060798645,
|
||
|
|
"metrics/advantage_var": 375.38916015625,
|
||
|
|
"regularization/lipschitz_norm": 1205.2469482421875,
|
||
|
|
"regularization/mmd": 0.1837475746870041,
|
||
|
|
"regularization/rkhs_norm": 218.74710083007812,
|
||
|
|
"regularization/w1": 1.0124075412750244,
|
||
|
|
"rewards/chosen": 0.5078481835631368,
|
||
|
|
"rewards/margins": 0.8058288991157312,
|
||
|
|
"rewards/rejected": -0.2979807155525944,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 15.182455062866211,
|
||
|
|
"kl": 13.751710891723633,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -34762249.721518986,
|
||
|
|
"logits/rejected": -36049483.85185185,
|
||
|
|
"logps/chosen": -489.95352056962025,
|
||
|
|
"logps/rejected": -373.3676456404321,
|
||
|
|
"loss": 0.5569,
|
||
|
|
"loss/base_kto": 3.205108404159546,
|
||
|
|
"metrics/advantage_var": 401.9645690917969,
|
||
|
|
"regularization/lipschitz_norm": 1263.1134033203125,
|
||
|
|
"regularization/mmd": 0.18922996520996094,
|
||
|
|
"regularization/rkhs_norm": 225.27378845214844,
|
||
|
|
"regularization/w1": 1.061015248298645,
|
||
|
|
"rewards/chosen": 0.4993312690831438,
|
||
|
|
"rewards/margins": 0.8156143655850004,
|
||
|
|
"rewards/rejected": -0.31628309650185665,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 14.551630973815918,
|
||
|
|
"kl": 12.297151565551758,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -35766256.12403101,
|
||
|
|
"logits/rejected": -35776399.11811024,
|
||
|
|
"logps/chosen": -490.0437015503876,
|
||
|
|
"logps/rejected": -368.00113188976377,
|
||
|
|
"loss": 0.558,
|
||
|
|
"loss/base_kto": 3.2429046630859375,
|
||
|
|
"metrics/advantage_var": 408.36981201171875,
|
||
|
|
"regularization/lipschitz_norm": 1229.8167724609375,
|
||
|
|
"regularization/mmd": 0.18843434751033783,
|
||
|
|
"regularization/rkhs_norm": 224.3266143798828,
|
||
|
|
"regularization/w1": 1.0330461263656616,
|
||
|
|
"rewards/chosen": 0.5068759652071221,
|
||
|
|
"rewards/margins": 0.8017690527594105,
|
||
|
|
"rewards/rejected": -0.2948930875522884,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 15.871299743652344,
|
||
|
|
"kl": 13.784205436706543,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -32945674.6496,
|
||
|
|
"logits/rejected": -35192819.49312977,
|
||
|
|
"logps/chosen": -451.0255,
|
||
|
|
"logps/rejected": -388.55708492366415,
|
||
|
|
"loss": 0.5493,
|
||
|
|
"loss/base_kto": 3.1717541217803955,
|
||
|
|
"metrics/advantage_var": 378.71173095703125,
|
||
|
|
"regularization/lipschitz_norm": 1236.295166015625,
|
||
|
|
"regularization/mmd": 0.18411408364772797,
|
||
|
|
"regularization/rkhs_norm": 219.18345642089844,
|
||
|
|
"regularization/w1": 1.0384880304336548,
|
||
|
|
"rewards/chosen": 0.5057248046875,
|
||
|
|
"rewards/margins": 0.8527784839426289,
|
||
|
|
"rewards/rejected": -0.3470536792551288,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 18.033416748046875,
|
||
|
|
"kl": 13.068841934204102,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -34780834.025316454,
|
||
|
|
"logits/rejected": -36113515.45679013,
|
||
|
|
"logps/chosen": -458.52037183544303,
|
||
|
|
"logps/rejected": -364.9637345679012,
|
||
|
|
"loss": 0.5534,
|
||
|
|
"loss/base_kto": 3.164045572280884,
|
||
|
|
"metrics/advantage_var": 413.14263916015625,
|
||
|
|
"regularization/lipschitz_norm": 1274.2684326171875,
|
||
|
|
"regularization/mmd": 0.19258885085582733,
|
||
|
|
"regularization/rkhs_norm": 229.2724609375,
|
||
|
|
"regularization/w1": 1.0703853368759155,
|
||
|
|
"rewards/chosen": 0.5223610793487935,
|
||
|
|
"rewards/margins": 0.877933485505357,
|
||
|
|
"rewards/rejected": -0.3555724061565635,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 18.45494842529297,
|
||
|
|
"kl": 13.948599815368652,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -34890844.25225225,
|
||
|
|
"logits/rejected": -36268308.84690554,
|
||
|
|
"logps/chosen": -501.97475600600603,
|
||
|
|
"logps/rejected": -373.98310260586317,
|
||
|
|
"loss": 0.5615,
|
||
|
|
"loss/base_kto": 3.147346258163452,
|
||
|
|
"metrics/advantage_var": 466.57666015625,
|
||
|
|
"regularization/lipschitz_norm": 1360.3629150390625,
|
||
|
|
"regularization/mmd": 0.20164580643177032,
|
||
|
|
"regularization/rkhs_norm": 240.05455017089844,
|
||
|
|
"regularization/w1": 1.1427048444747925,
|
||
|
|
"rewards/chosen": 0.5498876256627722,
|
||
|
|
"rewards/margins": 0.8893085034238675,
|
||
|
|
"rewards/rejected": -0.33942087776109525,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 14.303071022033691,
|
||
|
|
"kl": 12.555839538574219,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -33900531.27950311,
|
||
|
|
"logits/rejected": -34763418.56603774,
|
||
|
|
"logps/chosen": -474.8220108695652,
|
||
|
|
"logps/rejected": -383.9449685534591,
|
||
|
|
"loss": 0.5536,
|
||
|
|
"loss/base_kto": 3.195784568786621,
|
||
|
|
"metrics/advantage_var": 405.81805419921875,
|
||
|
|
"regularization/lipschitz_norm": 1243.735595703125,
|
||
|
|
"regularization/mmd": 0.18844656646251678,
|
||
|
|
"regularization/rkhs_norm": 224.34117126464844,
|
||
|
|
"regularization/w1": 1.0447378158569336,
|
||
|
|
"rewards/chosen": 0.528636482191382,
|
||
|
|
"rewards/margins": 0.8448140422898982,
|
||
|
|
"rewards/rejected": -0.3161775600985161,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 17.929088592529297,
|
||
|
|
"kl": 13.852238655090332,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -34382224.55520505,
|
||
|
|
"logits/rejected": -34066194.229102165,
|
||
|
|
"logps/chosen": -477.84118690851733,
|
||
|
|
"logps/rejected": -374.51456075851394,
|
||
|
|
"loss": 0.562,
|
||
|
|
"loss/base_kto": 3.195948839187622,
|
||
|
|
"metrics/advantage_var": 431.516845703125,
|
||
|
|
"regularization/lipschitz_norm": 1316.0577392578125,
|
||
|
|
"regularization/mmd": 0.19486133754253387,
|
||
|
|
"regularization/rkhs_norm": 231.97781372070312,
|
||
|
|
"regularization/w1": 1.1054884195327759,
|
||
|
|
"rewards/chosen": 0.5255168349208892,
|
||
|
|
"rewards/margins": 0.8452415644765683,
|
||
|
|
"rewards/rejected": -0.31972472955567915,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 17.644804000854492,
|
||
|
|
"kl": 13.09256649017334,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34767201.215189874,
|
||
|
|
"logits/rejected": -36606381.82716049,
|
||
|
|
"logps/chosen": -478.00820806962025,
|
||
|
|
"logps/rejected": -386.5558690200617,
|
||
|
|
"loss": 0.5583,
|
||
|
|
"loss/base_kto": 3.164849042892456,
|
||
|
|
"metrics/advantage_var": 428.4366149902344,
|
||
|
|
"regularization/lipschitz_norm": 1319.111572265625,
|
||
|
|
"regularization/mmd": 0.1931963413953781,
|
||
|
|
"regularization/rkhs_norm": 229.9956512451172,
|
||
|
|
"regularization/w1": 1.1080538034439087,
|
||
|
|
"rewards/chosen": 0.5406829254536689,
|
||
|
|
"rewards/margins": 0.8741082589241429,
|
||
|
|
"rewards/rejected": -0.33342533347047404,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 18.839326858520508,
|
||
|
|
"kl": 14.727572441101074,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -34945595.6108453,
|
||
|
|
"logits/rejected": -36285121.6661562,
|
||
|
|
"logps/chosen": -476.87764154704945,
|
||
|
|
"logps/rejected": -378.3577239663093,
|
||
|
|
"loss": 0.5696,
|
||
|
|
"loss/base_kto": 3.226821184158325,
|
||
|
|
"metrics/advantage_var": 441.74688720703125,
|
||
|
|
"regularization/lipschitz_norm": 1350.0693359375,
|
||
|
|
"regularization/mmd": 0.1956876814365387,
|
||
|
|
"regularization/rkhs_norm": 232.96153259277344,
|
||
|
|
"regularization/w1": 1.1340582370758057,
|
||
|
|
"rewards/chosen": 0.471017791894064,
|
||
|
|
"rewards/margins": 0.797263367078908,
|
||
|
|
"rewards/rejected": -0.326245575184844,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 17.33658218383789,
|
||
|
|
"kl": 13.067753791809082,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -36326757.06298003,
|
||
|
|
"logits/rejected": -36277031.47853736,
|
||
|
|
"logps/chosen": -482.0709485407066,
|
||
|
|
"logps/rejected": -391.3971333465819,
|
||
|
|
"loss": 0.5539,
|
||
|
|
"loss/base_kto": 3.179420232772827,
|
||
|
|
"metrics/advantage_var": 400.2893981933594,
|
||
|
|
"regularization/lipschitz_norm": 1266.2547607421875,
|
||
|
|
"regularization/mmd": 0.18828484416007996,
|
||
|
|
"regularization/rkhs_norm": 224.1486053466797,
|
||
|
|
"regularization/w1": 1.0636539459228516,
|
||
|
|
"rewards/chosen": 0.5448522142917147,
|
||
|
|
"rewards/margins": 0.8471926990564081,
|
||
|
|
"rewards/rejected": -0.30234048476469344,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 15.574108123779297,
|
||
|
|
"kl": 11.259978294372559,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -36309418.40729483,
|
||
|
|
"logits/rejected": -35371538.10932476,
|
||
|
|
"logps/chosen": -503.0312974924012,
|
||
|
|
"logps/rejected": -383.38972065916397,
|
||
|
|
"loss": 0.5635,
|
||
|
|
"loss/base_kto": 3.139202833175659,
|
||
|
|
"metrics/advantage_var": 470.6390686035156,
|
||
|
|
"regularization/lipschitz_norm": 1390.9105224609375,
|
||
|
|
"regularization/mmd": 0.2007589340209961,
|
||
|
|
"regularization/rkhs_norm": 238.9987335205078,
|
||
|
|
"regularization/w1": 1.1683648824691772,
|
||
|
|
"rewards/chosen": 0.5424724915107333,
|
||
|
|
"rewards/margins": 0.9181714519897828,
|
||
|
|
"rewards/rejected": -0.37569896047904944,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 11.174159049987793,
|
||
|
|
"kl": 12.35366439819336,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -34988064.2519685,
|
||
|
|
"logits/rejected": -35116671.80155039,
|
||
|
|
"logps/chosen": -463.37052165354334,
|
||
|
|
"logps/rejected": -388.6034399224806,
|
||
|
|
"loss": 0.5506,
|
||
|
|
"loss/base_kto": 3.1795132160186768,
|
||
|
|
"metrics/advantage_var": 378.6888427734375,
|
||
|
|
"regularization/lipschitz_norm": 1238.9239501953125,
|
||
|
|
"regularization/mmd": 0.18431107699871063,
|
||
|
|
"regularization/rkhs_norm": 219.41796875,
|
||
|
|
"regularization/w1": 1.0406960248947144,
|
||
|
|
"rewards/chosen": 0.47429660586860234,
|
||
|
|
"rewards/margins": 0.8348589998850753,
|
||
|
|
"rewards/rejected": -0.3605623940164729,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 22.472700119018555,
|
||
|
|
"kl": 13.033785820007324,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -35498326.998373985,
|
||
|
|
"logits/rejected": -37163365.245112784,
|
||
|
|
"logps/chosen": -489.7123983739837,
|
||
|
|
"logps/rejected": -376.54320958646616,
|
||
|
|
"loss": 0.5607,
|
||
|
|
"loss/base_kto": 3.1383798122406006,
|
||
|
|
"metrics/advantage_var": 430.47833251953125,
|
||
|
|
"regularization/lipschitz_norm": 1368.0810546875,
|
||
|
|
"regularization/mmd": 0.19792315363883972,
|
||
|
|
"regularization/rkhs_norm": 235.622802734375,
|
||
|
|
"regularization/w1": 1.1491881608963013,
|
||
|
|
"rewards/chosen": 0.5409264045033029,
|
||
|
|
"rewards/margins": 0.9076404001858687,
|
||
|
|
"rewards/rejected": -0.3667139956825658,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 20.603042602539062,
|
||
|
|
"kl": 13.964932441711426,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -33562544.12480499,
|
||
|
|
"logits/rejected": -35524364.41940532,
|
||
|
|
"logps/chosen": -480.97723283931356,
|
||
|
|
"logps/rejected": -365.04161776212834,
|
||
|
|
"loss": 0.5519,
|
||
|
|
"loss/base_kto": 3.180474042892456,
|
||
|
|
"metrics/advantage_var": 403.8537292480469,
|
||
|
|
"regularization/lipschitz_norm": 1242.5142822265625,
|
||
|
|
"regularization/mmd": 0.19066281616687775,
|
||
|
|
"regularization/rkhs_norm": 226.9795379638672,
|
||
|
|
"regularization/w1": 1.0437120199203491,
|
||
|
|
"rewards/chosen": 0.5153949041262432,
|
||
|
|
"rewards/margins": 0.8494487197500207,
|
||
|
|
"rewards/rejected": -0.3340538156237774,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 14.573284149169922,
|
||
|
|
"kl": 13.512062072753906,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -34238379.6133122,
|
||
|
|
"logits/rejected": -35476383.753466874,
|
||
|
|
"logps/chosen": -480.55923137876385,
|
||
|
|
"logps/rejected": -369.87403697996916,
|
||
|
|
"loss": 0.5594,
|
||
|
|
"loss/base_kto": 3.1754276752471924,
|
||
|
|
"metrics/advantage_var": 431.06494140625,
|
||
|
|
"regularization/lipschitz_norm": 1316.7503662109375,
|
||
|
|
"regularization/mmd": 0.19363752007484436,
|
||
|
|
"regularization/rkhs_norm": 230.5208740234375,
|
||
|
|
"regularization/w1": 1.1060702800750732,
|
||
|
|
"rewards/chosen": 0.5244395018757428,
|
||
|
|
"rewards/margins": 0.8500996314169842,
|
||
|
|
"rewards/rejected": -0.32566012954124135,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.978042558275912e+17,
|
||
|
|
"train_loss": 0.5867222310759654,
|
||
|
|
"train_runtime": 11075.4163,
|
||
|
|
"train_samples_per_second": 13.383,
|
||
|
|
"train_steps_per_second": 0.209
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.978042558275912e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|