2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 16.537830352783203,
|
||
|
|
"kl": 20.99601173400879,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -36331023.31968504,
|
||
|
|
"logits/rejected": -37604871.14418605,
|
||
|
|
"logps/chosen": -479.68499015748034,
|
||
|
|
"logps/rejected": -353.2168362403101,
|
||
|
|
"loss": 0.5789,
|
||
|
|
"loss/base_kto": 3.9304776191711426,
|
||
|
|
"loss/total_with_kl": 4.631022930145264,
|
||
|
|
"metrics/advantage_var": 420.7478942871094,
|
||
|
|
"regularization/advantage_var": 420.7478942871094,
|
||
|
|
"regularization/kl": 0.7005452513694763,
|
||
|
|
"rewards/chosen": 0.28216278797059546,
|
||
|
|
"rewards/margins": 0.06821970869491717,
|
||
|
|
"rewards/rejected": 0.2139430792756783,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 15.986422538757324,
|
||
|
|
"kl": 5.433504581451416,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -36267776.0,
|
||
|
|
"logits/rejected": -36990547.2,
|
||
|
|
"logps/chosen": -476.61943359375,
|
||
|
|
"logps/rejected": -412.352294921875,
|
||
|
|
"loss": 0.5335,
|
||
|
|
"loss/base_kto": 3.932533025741577,
|
||
|
|
"loss/total_with_kl": 4.268146991729736,
|
||
|
|
"metrics/advantage_var": 201.5700225830078,
|
||
|
|
"regularization/advantage_var": 201.5700225830078,
|
||
|
|
"regularization/kl": 0.33561408519744873,
|
||
|
|
"rewards/chosen": -0.014302882552146911,
|
||
|
|
"rewards/margins": 0.057395032048225395,
|
||
|
|
"rewards/rejected": -0.07169791460037231,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 15.14694595336914,
|
||
|
|
"kl": 4.5898332595825195,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -36111591.8490566,
|
||
|
|
"logits/rejected": -38411314.88198758,
|
||
|
|
"logps/chosen": -487.9725825471698,
|
||
|
|
"logps/rejected": -374.0280958850932,
|
||
|
|
"loss": 0.5208,
|
||
|
|
"loss/base_kto": 3.8561952114105225,
|
||
|
|
"loss/total_with_kl": 4.166226387023926,
|
||
|
|
"metrics/advantage_var": 186.20510864257812,
|
||
|
|
"regularization/advantage_var": 186.20510864257812,
|
||
|
|
"regularization/kl": 0.3100314736366272,
|
||
|
|
"rewards/chosen": 0.0318767859500909,
|
||
|
|
"rewards/margins": 0.15225735341782187,
|
||
|
|
"rewards/rejected": -0.12038056746773097,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 16.574857711791992,
|
||
|
|
"kl": 10.953143119812012,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -37778135.748427674,
|
||
|
|
"logits/rejected": -37664049.29192547,
|
||
|
|
"logps/chosen": -503.7589426100629,
|
||
|
|
"logps/rejected": -382.98825698757764,
|
||
|
|
"loss": 0.5201,
|
||
|
|
"loss/base_kto": 3.8481743335723877,
|
||
|
|
"loss/total_with_kl": 4.160851001739502,
|
||
|
|
"metrics/advantage_var": 187.7936553955078,
|
||
|
|
"regularization/advantage_var": 187.7936553955078,
|
||
|
|
"regularization/kl": 0.31267642974853516,
|
||
|
|
"rewards/chosen": 0.19398306600702633,
|
||
|
|
"rewards/margins": 0.13317571944978823,
|
||
|
|
"rewards/rejected": 0.06080734655723809,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 29.886825561523438,
|
||
|
|
"kl": 14.196418762207031,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -36849322.666666664,
|
||
|
|
"logits/rejected": -37452472.57680251,
|
||
|
|
"logps/chosen": -506.85835280373834,
|
||
|
|
"logps/rejected": -378.2287911442006,
|
||
|
|
"loss": 0.5202,
|
||
|
|
"loss/base_kto": 3.798060178756714,
|
||
|
|
"loss/total_with_kl": 4.1617608070373535,
|
||
|
|
"metrics/advantage_var": 218.4387664794922,
|
||
|
|
"regularization/advantage_var": 218.4387664794922,
|
||
|
|
"regularization/kl": 0.36370059847831726,
|
||
|
|
"rewards/chosen": 0.2566196049485251,
|
||
|
|
"rewards/margins": 0.1754785187408632,
|
||
|
|
"rewards/rejected": 0.08114108620766189,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 14.523025512695312,
|
||
|
|
"kl": 8.677205085754395,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -36201331.14064915,
|
||
|
|
"logits/rejected": -37661542.319115326,
|
||
|
|
"logps/chosen": -468.98000386398763,
|
||
|
|
"logps/rejected": -370.2740422590837,
|
||
|
|
"loss": 0.5248,
|
||
|
|
"loss/base_kto": 3.798210859298706,
|
||
|
|
"loss/total_with_kl": 4.1985764503479,
|
||
|
|
"metrics/advantage_var": 240.4597930908203,
|
||
|
|
"regularization/advantage_var": 240.4597930908203,
|
||
|
|
"regularization/kl": 0.40036556124687195,
|
||
|
|
"rewards/chosen": 0.13418971773645794,
|
||
|
|
"rewards/margins": 0.18964537603291948,
|
||
|
|
"rewards/rejected": -0.05545565829646154,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 17.636444091796875,
|
||
|
|
"kl": 9.173022270202637,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -36726727.633027524,
|
||
|
|
"logits/rejected": -38022880.10223642,
|
||
|
|
"logps/chosen": -479.09250764525996,
|
||
|
|
"logps/rejected": -384.2581619408946,
|
||
|
|
"loss": 0.5267,
|
||
|
|
"loss/base_kto": 3.7889816761016846,
|
||
|
|
"loss/total_with_kl": 4.213504314422607,
|
||
|
|
"metrics/advantage_var": 254.96865844726562,
|
||
|
|
"regularization/advantage_var": 254.96865844726562,
|
||
|
|
"regularization/kl": 0.4245227873325348,
|
||
|
|
"rewards/chosen": 0.1997123974908018,
|
||
|
|
"rewards/margins": 0.19239941792973037,
|
||
|
|
"rewards/rejected": 0.007312979561071426,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 15.215126991271973,
|
||
|
|
"kl": 12.488289833068848,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36265196.307692304,
|
||
|
|
"logits/rejected": -37743466.463492066,
|
||
|
|
"logps/chosen": -493.04855769230767,
|
||
|
|
"logps/rejected": -360.37276785714283,
|
||
|
|
"loss": 0.5221,
|
||
|
|
"loss/base_kto": 3.7549118995666504,
|
||
|
|
"loss/total_with_kl": 4.176564693450928,
|
||
|
|
"metrics/advantage_var": 253.24485778808594,
|
||
|
|
"regularization/advantage_var": 253.24485778808594,
|
||
|
|
"regularization/kl": 0.421652615070343,
|
||
|
|
"rewards/chosen": 0.23179610032301684,
|
||
|
|
"rewards/margins": 0.23726204985081786,
|
||
|
|
"rewards/rejected": -0.005465949527801029,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 17.90062141418457,
|
||
|
|
"kl": 11.389983177185059,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -36015610.40873635,
|
||
|
|
"logits/rejected": -36947951.97496088,
|
||
|
|
"logps/chosen": -525.790025351014,
|
||
|
|
"logps/rejected": -366.84952073552427,
|
||
|
|
"loss": 0.5205,
|
||
|
|
"loss/base_kto": 3.7394680976867676,
|
||
|
|
"loss/total_with_kl": 4.1636176109313965,
|
||
|
|
"metrics/advantage_var": 254.74453735351562,
|
||
|
|
"regularization/advantage_var": 254.74453735351562,
|
||
|
|
"regularization/kl": 0.42414966225624084,
|
||
|
|
"rewards/chosen": 0.2547431743461145,
|
||
|
|
"rewards/margins": 0.23224709267897406,
|
||
|
|
"rewards/rejected": 0.02249608166714043,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 14.606820106506348,
|
||
|
|
"kl": 13.57861328125,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -36329533.27874016,
|
||
|
|
"logits/rejected": -37751955.64651163,
|
||
|
|
"logps/chosen": -460.625,
|
||
|
|
"logps/rejected": -373.60237403100774,
|
||
|
|
"loss": 0.5201,
|
||
|
|
"loss/base_kto": 3.7257869243621826,
|
||
|
|
"loss/total_with_kl": 4.161072254180908,
|
||
|
|
"metrics/advantage_var": 261.4327392578125,
|
||
|
|
"regularization/advantage_var": 261.4327392578125,
|
||
|
|
"regularization/kl": 0.4352855384349823,
|
||
|
|
"rewards/chosen": 0.25165583092396654,
|
||
|
|
"rewards/margins": 0.2519974947281317,
|
||
|
|
"rewards/rejected": -0.0003416638041651526,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 15.993019104003906,
|
||
|
|
"kl": 10.934139251708984,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37423401.18611988,
|
||
|
|
"logits/rejected": -38687883.49226006,
|
||
|
|
"logps/chosen": -475.56269716088326,
|
||
|
|
"logps/rejected": -378.687935371517,
|
||
|
|
"loss": 0.5129,
|
||
|
|
"loss/base_kto": 3.7245919704437256,
|
||
|
|
"loss/total_with_kl": 4.102898597717285,
|
||
|
|
"metrics/advantage_var": 227.2112274169922,
|
||
|
|
"regularization/advantage_var": 227.2112274169922,
|
||
|
|
"regularization/kl": 0.37830671668052673,
|
||
|
|
"rewards/chosen": 0.23485413560355875,
|
||
|
|
"rewards/margins": 0.2576988723936991,
|
||
|
|
"rewards/rejected": -0.022844736790140346,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 14.208782196044922,
|
||
|
|
"kl": 3.9187302589416504,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -35056635.0531401,
|
||
|
|
"logits/rejected": -37595947.119878605,
|
||
|
|
"logps/chosen": -482.7842190016103,
|
||
|
|
"logps/rejected": -351.66445371775416,
|
||
|
|
"loss": 0.5095,
|
||
|
|
"loss/base_kto": 3.697678327560425,
|
||
|
|
"loss/total_with_kl": 4.075837135314941,
|
||
|
|
"metrics/advantage_var": 227.122314453125,
|
||
|
|
"regularization/advantage_var": 227.122314453125,
|
||
|
|
"regularization/kl": 0.3781586289405823,
|
||
|
|
"rewards/chosen": 0.04967179352150639,
|
||
|
|
"rewards/margins": 0.2854993654077687,
|
||
|
|
"rewards/rejected": -0.23582757188626233,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 16.923664093017578,
|
||
|
|
"kl": 5.567387580871582,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -37492409.741324924,
|
||
|
|
"logits/rejected": -37807310.06811146,
|
||
|
|
"logps/chosen": -484.5050276025237,
|
||
|
|
"logps/rejected": -390.71084075077397,
|
||
|
|
"loss": 0.5169,
|
||
|
|
"loss/base_kto": 3.747518539428711,
|
||
|
|
"loss/total_with_kl": 4.134842872619629,
|
||
|
|
"metrics/advantage_var": 232.62698364257812,
|
||
|
|
"regularization/advantage_var": 232.62698364257812,
|
||
|
|
"regularization/kl": 0.38732391595840454,
|
||
|
|
"rewards/chosen": 0.03686676867752797,
|
||
|
|
"rewards/margins": 0.24184379829488478,
|
||
|
|
"rewards/rejected": -0.2049770296173568,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 13.19844913482666,
|
||
|
|
"kl": 10.004745483398438,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -37259758.77675841,
|
||
|
|
"logits/rejected": -38464577.4313099,
|
||
|
|
"logps/chosen": -492.9337729357798,
|
||
|
|
"logps/rejected": -376.2843949680511,
|
||
|
|
"loss": 0.5165,
|
||
|
|
"loss/base_kto": 3.7302138805389404,
|
||
|
|
"loss/total_with_kl": 4.13217306137085,
|
||
|
|
"metrics/advantage_var": 241.4169921875,
|
||
|
|
"regularization/advantage_var": 241.4169921875,
|
||
|
|
"regularization/kl": 0.40195927023887634,
|
||
|
|
"rewards/chosen": 0.22754428875191132,
|
||
|
|
"rewards/margins": 0.26821775490723465,
|
||
|
|
"rewards/rejected": -0.04067346615532336,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 10.370858192443848,
|
||
|
|
"kl": 8.129781723022461,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -35331912.4992,
|
||
|
|
"logits/rejected": -36473516.751145035,
|
||
|
|
"logps/chosen": -496.1955,
|
||
|
|
"logps/rejected": -397.9780057251908,
|
||
|
|
"loss": 0.5201,
|
||
|
|
"loss/base_kto": 3.7339470386505127,
|
||
|
|
"loss/total_with_kl": 4.160952091217041,
|
||
|
|
"metrics/advantage_var": 256.45947265625,
|
||
|
|
"regularization/advantage_var": 256.45947265625,
|
||
|
|
"regularization/kl": 0.4270050525665283,
|
||
|
|
"rewards/chosen": 0.1226624755859375,
|
||
|
|
"rewards/margins": 0.256651463166448,
|
||
|
|
"rewards/rejected": -0.1339889875805105,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 18.85415267944336,
|
||
|
|
"kl": 6.334065914154053,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -35705573.51724138,
|
||
|
|
"logits/rejected": -36782858.367601246,
|
||
|
|
"logps/chosen": -471.8168593260188,
|
||
|
|
"logps/rejected": -366.6216413551402,
|
||
|
|
"loss": 0.5193,
|
||
|
|
"loss/base_kto": 3.683685302734375,
|
||
|
|
"loss/total_with_kl": 4.1541266441345215,
|
||
|
|
"metrics/advantage_var": 282.54736328125,
|
||
|
|
"regularization/advantage_var": 282.54736328125,
|
||
|
|
"regularization/kl": 0.47044140100479126,
|
||
|
|
"rewards/chosen": 0.08399816291832998,
|
||
|
|
"rewards/margins": 0.31805401179756476,
|
||
|
|
"rewards/rejected": -0.2340558488792348,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 15.948773384094238,
|
||
|
|
"kl": 14.533068656921387,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -36502211.156950675,
|
||
|
|
"logits/rejected": -37504350.271685764,
|
||
|
|
"logps/chosen": -455.46337817638266,
|
||
|
|
"logps/rejected": -359.12193126022913,
|
||
|
|
"loss": 0.5115,
|
||
|
|
"loss/base_kto": 3.6408157348632812,
|
||
|
|
"loss/total_with_kl": 4.092252254486084,
|
||
|
|
"metrics/advantage_var": 271.1326904296875,
|
||
|
|
"regularization/advantage_var": 271.1326904296875,
|
||
|
|
"regularization/kl": 0.45143595337867737,
|
||
|
|
"rewards/chosen": 0.28905634174432454,
|
||
|
|
"rewards/margins": 0.3318007051326608,
|
||
|
|
"rewards/rejected": -0.04274436338833623,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 16.080219268798828,
|
||
|
|
"kl": 13.947799682617188,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -35981587.692307696,
|
||
|
|
"logits/rejected": -36368252.87804878,
|
||
|
|
"logps/chosen": -483.78074919871796,
|
||
|
|
"logps/rejected": -374.1924066310976,
|
||
|
|
"loss": 0.5124,
|
||
|
|
"loss/base_kto": 3.667691946029663,
|
||
|
|
"loss/total_with_kl": 4.099027633666992,
|
||
|
|
"metrics/advantage_var": 259.0605163574219,
|
||
|
|
"regularization/advantage_var": 259.0605163574219,
|
||
|
|
"regularization/kl": 0.43133577704429626,
|
||
|
|
"rewards/chosen": 0.25421406672551083,
|
||
|
|
"rewards/margins": 0.29900450039983467,
|
||
|
|
"rewards/rejected": -0.04479043367432385,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 21.614364624023438,
|
||
|
|
"kl": 12.22073745727539,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -37742965.21898928,
|
||
|
|
"logits/rejected": -37564559.71929824,
|
||
|
|
"logps/chosen": -482.3780627871363,
|
||
|
|
"logps/rejected": -372.8219946172249,
|
||
|
|
"loss": 0.5222,
|
||
|
|
"loss/base_kto": 3.696674346923828,
|
||
|
|
"loss/total_with_kl": 4.177842140197754,
|
||
|
|
"metrics/advantage_var": 288.9895935058594,
|
||
|
|
"regularization/advantage_var": 288.9895935058594,
|
||
|
|
"regularization/kl": 0.4811677038669586,
|
||
|
|
"rewards/chosen": 0.20185038871823555,
|
||
|
|
"rewards/margins": 0.28784650850795535,
|
||
|
|
"rewards/rejected": -0.08599611978971977,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 25.629762649536133,
|
||
|
|
"kl": 5.442298412322998,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -36602081.3136289,
|
||
|
|
"logits/rejected": -38403513.03725783,
|
||
|
|
"logps/chosen": -492.5006670771757,
|
||
|
|
"logps/rejected": -376.49224571535024,
|
||
|
|
"loss": 0.5199,
|
||
|
|
"loss/base_kto": 3.730147123336792,
|
||
|
|
"loss/total_with_kl": 4.159346580505371,
|
||
|
|
"metrics/advantage_var": 257.77734375,
|
||
|
|
"regularization/advantage_var": 257.77734375,
|
||
|
|
"regularization/kl": 0.4291992783546448,
|
||
|
|
"rewards/chosen": 0.03159951459010834,
|
||
|
|
"rewards/margins": 0.25040941272779926,
|
||
|
|
"rewards/rejected": -0.21880989813769094,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 15.508956909179688,
|
||
|
|
"kl": 6.528402805328369,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36286199.741935484,
|
||
|
|
"logits/rejected": -36577885.60890302,
|
||
|
|
"logps/chosen": -490.13608870967744,
|
||
|
|
"logps/rejected": -368.38235294117646,
|
||
|
|
"loss": 0.5175,
|
||
|
|
"loss/base_kto": 3.636345624923706,
|
||
|
|
"loss/total_with_kl": 4.1403422355651855,
|
||
|
|
"metrics/advantage_var": 302.70098876953125,
|
||
|
|
"regularization/advantage_var": 302.70098876953125,
|
||
|
|
"regularization/kl": 0.5039971470832825,
|
||
|
|
"rewards/chosen": 0.15539588283650155,
|
||
|
|
"rewards/margins": 0.3653011706352148,
|
||
|
|
"rewards/rejected": -0.20990528779871323,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 15.6985502243042,
|
||
|
|
"kl": 8.336493492126465,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -35088697.65765766,
|
||
|
|
"logits/rejected": -38028051.17915309,
|
||
|
|
"logps/chosen": -507.7996433933934,
|
||
|
|
"logps/rejected": -362.4643729641694,
|
||
|
|
"loss": 0.5211,
|
||
|
|
"loss/base_kto": 3.701464891433716,
|
||
|
|
"loss/total_with_kl": 4.16871976852417,
|
||
|
|
"metrics/advantage_var": 280.633544921875,
|
||
|
|
"regularization/advantage_var": 280.633544921875,
|
||
|
|
"regularization/kl": 0.4672548472881317,
|
||
|
|
"rewards/chosen": 0.14615589863545186,
|
||
|
|
"rewards/margins": 0.3055379266249419,
|
||
|
|
"rewards/rejected": -0.15938202798949003,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 16.868915557861328,
|
||
|
|
"kl": 15.688308715820312,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -35149326.01610306,
|
||
|
|
"logits/rejected": -36754408.69195751,
|
||
|
|
"logps/chosen": -472.32352053140096,
|
||
|
|
"logps/rejected": -353.294812215478,
|
||
|
|
"loss": 0.5178,
|
||
|
|
"loss/base_kto": 3.7071945667266846,
|
||
|
|
"loss/total_with_kl": 4.142777919769287,
|
||
|
|
"metrics/advantage_var": 261.61181640625,
|
||
|
|
"regularization/advantage_var": 261.61181640625,
|
||
|
|
"regularization/kl": 0.4355836510658264,
|
||
|
|
"rewards/chosen": 0.2514052828728865,
|
||
|
|
"rewards/margins": 0.2728425848559456,
|
||
|
|
"rewards/rejected": -0.021437301983059084,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 11.048973083496094,
|
||
|
|
"kl": 5.838799476623535,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -36868629.40192926,
|
||
|
|
"logits/rejected": -37682253.811550155,
|
||
|
|
"logps/chosen": -484.14328778135047,
|
||
|
|
"logps/rejected": -388.22701367781156,
|
||
|
|
"loss": 0.5122,
|
||
|
|
"loss/base_kto": 3.668853759765625,
|
||
|
|
"loss/total_with_kl": 4.09730863571167,
|
||
|
|
"metrics/advantage_var": 257.3302307128906,
|
||
|
|
"regularization/advantage_var": 257.3302307128906,
|
||
|
|
"regularization/kl": 0.42845478653907776,
|
||
|
|
"rewards/chosen": 0.09705469309325387,
|
||
|
|
"rewards/margins": 0.313090947605032,
|
||
|
|
"rewards/rejected": -0.2160362545117781,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 26.0384464263916,
|
||
|
|
"kl": 7.713786602020264,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36236119.9752322,
|
||
|
|
"logits/rejected": -37724476.56782334,
|
||
|
|
"logps/chosen": -498.0817530959752,
|
||
|
|
"logps/rejected": -356.036597988959,
|
||
|
|
"loss": 0.5166,
|
||
|
|
"loss/base_kto": 3.686602830886841,
|
||
|
|
"loss/total_with_kl": 4.132496356964111,
|
||
|
|
"metrics/advantage_var": 267.8038024902344,
|
||
|
|
"regularization/advantage_var": 267.8038024902344,
|
||
|
|
"regularization/kl": 0.4458933472633362,
|
||
|
|
"rewards/chosen": 0.1319815573677559,
|
||
|
|
"rewards/margins": 0.2882677789411614,
|
||
|
|
"rewards/rejected": -0.15628622157340546,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 11.767626762390137,
|
||
|
|
"kl": 10.853642463684082,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -36151056.17257319,
|
||
|
|
"logits/rejected": -36493139.980982564,
|
||
|
|
"logps/chosen": -485.22404661016947,
|
||
|
|
"logps/rejected": -367.3628169572108,
|
||
|
|
"loss": 0.514,
|
||
|
|
"loss/base_kto": 3.650493621826172,
|
||
|
|
"loss/total_with_kl": 4.112260341644287,
|
||
|
|
"metrics/advantage_var": 277.3372497558594,
|
||
|
|
"regularization/advantage_var": 277.3372497558594,
|
||
|
|
"regularization/kl": 0.4617665410041809,
|
||
|
|
"rewards/chosen": 0.22506149597638192,
|
||
|
|
"rewards/margins": 0.32605570885694724,
|
||
|
|
"rewards/rejected": -0.10099421288056533,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 17.659984588623047,
|
||
|
|
"kl": 20.0924072265625,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -37527906.69879518,
|
||
|
|
"logits/rejected": -37452075.22077922,
|
||
|
|
"logps/chosen": -479.65342620481925,
|
||
|
|
"logps/rejected": -357.5139255275974,
|
||
|
|
"loss": 0.5092,
|
||
|
|
"loss/base_kto": 3.613422155380249,
|
||
|
|
"loss/total_with_kl": 4.073502540588379,
|
||
|
|
"metrics/advantage_var": 276.324462890625,
|
||
|
|
"regularization/advantage_var": 276.324462890625,
|
||
|
|
"regularization/kl": 0.46008023619651794,
|
||
|
|
"rewards/chosen": 0.37712696948683405,
|
||
|
|
"rewards/margins": 0.3512136504564687,
|
||
|
|
"rewards/rejected": 0.025913319030365388,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 17.75587272644043,
|
||
|
|
"kl": 19.05929183959961,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -36423829.853658535,
|
||
|
|
"logits/rejected": -38204685.12820513,
|
||
|
|
"logps/chosen": -465.6052305640244,
|
||
|
|
"logps/rejected": -347.2238581730769,
|
||
|
|
"loss": 0.5168,
|
||
|
|
"loss/base_kto": 3.6852288246154785,
|
||
|
|
"loss/total_with_kl": 4.134213447570801,
|
||
|
|
"metrics/advantage_var": 269.6606140136719,
|
||
|
|
"regularization/advantage_var": 269.6606140136719,
|
||
|
|
"regularization/kl": 0.44898486137390137,
|
||
|
|
"rewards/chosen": 0.3508343347689001,
|
||
|
|
"rewards/margins": 0.277042836975947,
|
||
|
|
"rewards/rejected": 0.0737914977929531,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 16.573986053466797,
|
||
|
|
"kl": 24.6936092376709,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -38083644.23529412,
|
||
|
|
"logits/rejected": -39725339.306666665,
|
||
|
|
"logps/chosen": -472.7370404411765,
|
||
|
|
"logps/rejected": -384.35723958333335,
|
||
|
|
"loss": 0.5203,
|
||
|
|
"loss/base_kto": 3.684603214263916,
|
||
|
|
"loss/total_with_kl": 4.162057399749756,
|
||
|
|
"metrics/advantage_var": 286.75927734375,
|
||
|
|
"regularization/advantage_var": 286.75927734375,
|
||
|
|
"regularization/kl": 0.47745418548583984,
|
||
|
|
"rewards/chosen": 0.43507479499368107,
|
||
|
|
"rewards/margins": 0.24651265611835554,
|
||
|
|
"rewards/rejected": 0.18856213887532552,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 17.07406997680664,
|
||
|
|
"kl": 24.37955665588379,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -37937413.51384615,
|
||
|
|
"logits/rejected": -37525526.755555555,
|
||
|
|
"logps/chosen": -448.4246153846154,
|
||
|
|
"logps/rejected": -393.0105406746032,
|
||
|
|
"loss": 0.5137,
|
||
|
|
"loss/base_kto": 3.678205966949463,
|
||
|
|
"loss/total_with_kl": 4.109408855438232,
|
||
|
|
"metrics/advantage_var": 258.98077392578125,
|
||
|
|
"regularization/advantage_var": 258.98077392578125,
|
||
|
|
"regularization/kl": 0.4312029778957367,
|
||
|
|
"rewards/chosen": 0.434171142578125,
|
||
|
|
"rewards/margins": 0.2800571792844742,
|
||
|
|
"rewards/rejected": 0.1541139632936508,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 27.32758140563965,
|
||
|
|
"kl": 25.332792282104492,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36482125.283018865,
|
||
|
|
"logits/rejected": -37889904.89440994,
|
||
|
|
"logps/chosen": -484.06348270440253,
|
||
|
|
"logps/rejected": -386.7630046583851,
|
||
|
|
"loss": 0.516,
|
||
|
|
"loss/base_kto": 3.6858062744140625,
|
||
|
|
"loss/total_with_kl": 4.127625942230225,
|
||
|
|
"metrics/advantage_var": 265.3571472167969,
|
||
|
|
"regularization/advantage_var": 265.3571472167969,
|
||
|
|
"regularization/kl": 0.4418196380138397,
|
||
|
|
"rewards/chosen": 0.4418405976685338,
|
||
|
|
"rewards/margins": 0.2693183549032848,
|
||
|
|
"rewards/rejected": 0.17252224276524894,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 11.3949556350708,
|
||
|
|
"kl": 28.55071449279785,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -36017894.77221325,
|
||
|
|
"logits/rejected": -37295334.82602118,
|
||
|
|
"logps/chosen": -490.53130048465266,
|
||
|
|
"logps/rejected": -365.21683528744325,
|
||
|
|
"loss": 0.5115,
|
||
|
|
"loss/base_kto": 3.6491081714630127,
|
||
|
|
"loss/total_with_kl": 4.091750621795654,
|
||
|
|
"metrics/advantage_var": 265.85137939453125,
|
||
|
|
"regularization/advantage_var": 265.85137939453125,
|
||
|
|
"regularization/kl": 0.4426425099372864,
|
||
|
|
"rewards/chosen": 0.4128284361905543,
|
||
|
|
"rewards/margins": 0.28994236712793614,
|
||
|
|
"rewards/rejected": 0.12288606906261819,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 15.193487167358398,
|
||
|
|
"kl": 19.492938995361328,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -37432749.01529052,
|
||
|
|
"logits/rejected": -38655659.7571885,
|
||
|
|
"logps/chosen": -493.74522171253824,
|
||
|
|
"logps/rejected": -369.65869608626195,
|
||
|
|
"loss": 0.5145,
|
||
|
|
"loss/base_kto": 3.6358978748321533,
|
||
|
|
"loss/total_with_kl": 4.1160454750061035,
|
||
|
|
"metrics/advantage_var": 288.37677001953125,
|
||
|
|
"regularization/advantage_var": 288.37677001953125,
|
||
|
|
"regularization/kl": 0.48014727234840393,
|
||
|
|
"rewards/chosen": 0.37165909513421014,
|
||
|
|
"rewards/margins": 0.32228481639230655,
|
||
|
|
"rewards/rejected": 0.04937427874190358,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 26.08849334716797,
|
||
|
|
"kl": 22.695865631103516,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -35710002.7904,
|
||
|
|
"logits/rejected": -37808418.784732826,
|
||
|
|
"logps/chosen": -471.93155,
|
||
|
|
"logps/rejected": -374.5293893129771,
|
||
|
|
"loss": 0.5082,
|
||
|
|
"loss/base_kto": 3.6345341205596924,
|
||
|
|
"loss/total_with_kl": 4.0658087730407715,
|
||
|
|
"metrics/advantage_var": 259.023681640625,
|
||
|
|
"regularization/advantage_var": 259.023681640625,
|
||
|
|
"regularization/kl": 0.4312744140625,
|
||
|
|
"rewards/chosen": 0.334289599609375,
|
||
|
|
"rewards/margins": 0.29120993510238996,
|
||
|
|
"rewards/rejected": 0.043079664506985034,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 11.127323150634766,
|
||
|
|
"kl": 14.99573802947998,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -35764466.095846646,
|
||
|
|
"logits/rejected": -38377709.99388379,
|
||
|
|
"logps/chosen": -483.01747204472844,
|
||
|
|
"logps/rejected": -362.012996941896,
|
||
|
|
"loss": 0.5077,
|
||
|
|
"loss/base_kto": 3.615683078765869,
|
||
|
|
"loss/total_with_kl": 4.061391353607178,
|
||
|
|
"metrics/advantage_var": 267.6928405761719,
|
||
|
|
"regularization/advantage_var": 267.6928405761719,
|
||
|
|
"regularization/kl": 0.44570857286453247,
|
||
|
|
"rewards/chosen": 0.30560897485897565,
|
||
|
|
"rewards/margins": 0.3548047515154735,
|
||
|
|
"rewards/rejected": -0.049195776656497874,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 12.544903755187988,
|
||
|
|
"kl": 5.25020170211792,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -37378966.51664025,
|
||
|
|
"logits/rejected": -38261032.62865948,
|
||
|
|
"logps/chosen": -484.17670364500793,
|
||
|
|
"logps/rejected": -372.1931818181818,
|
||
|
|
"loss": 0.5267,
|
||
|
|
"loss/base_kto": 3.656712770462036,
|
||
|
|
"loss/total_with_kl": 4.213390827178955,
|
||
|
|
"metrics/advantage_var": 334.3412170410156,
|
||
|
|
"regularization/advantage_var": 334.3412170410156,
|
||
|
|
"regularization/kl": 0.5566781163215637,
|
||
|
|
"rewards/chosen": 0.03584157759338476,
|
||
|
|
"rewards/margins": 0.32360517611334333,
|
||
|
|
"rewards/rejected": -0.2877635985199586,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 14.217246055603027,
|
||
|
|
"kl": 18.524065017700195,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -38396444.981132075,
|
||
|
|
"logits/rejected": -39524243.875776395,
|
||
|
|
"logps/chosen": -514.2330483490566,
|
||
|
|
"logps/rejected": -352.86781832298135,
|
||
|
|
"loss": 0.5154,
|
||
|
|
"loss/base_kto": 3.6119332313537598,
|
||
|
|
"loss/total_with_kl": 4.1233439445495605,
|
||
|
|
"metrics/advantage_var": 307.15313720703125,
|
||
|
|
"regularization/advantage_var": 307.15313720703125,
|
||
|
|
"regularization/kl": 0.5114099383354187,
|
||
|
|
"rewards/chosen": 0.311815729681051,
|
||
|
|
"rewards/margins": 0.3291521204547531,
|
||
|
|
"rewards/rejected": -0.017336390773702113,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 16.856353759765625,
|
||
|
|
"kl": 9.3283109664917,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -38511310.596491225,
|
||
|
|
"logits/rejected": -37825042.03369066,
|
||
|
|
"logps/chosen": -489.4402412280702,
|
||
|
|
"logps/rejected": -397.62035796324653,
|
||
|
|
"loss": 0.5151,
|
||
|
|
"loss/base_kto": 3.7072436809539795,
|
||
|
|
"loss/total_with_kl": 4.120835781097412,
|
||
|
|
"metrics/advantage_var": 248.4034881591797,
|
||
|
|
"regularization/advantage_var": 248.4034881591797,
|
||
|
|
"regularization/kl": 0.41359183192253113,
|
||
|
|
"rewards/chosen": 0.10794661185768042,
|
||
|
|
"rewards/margins": 0.2651011799792707,
|
||
|
|
"rewards/rejected": -0.15715456812159026,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 17.514570236206055,
|
||
|
|
"kl": 9.719375610351562,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -38009808.73846154,
|
||
|
|
"logits/rejected": -38305022.36942675,
|
||
|
|
"logps/chosen": -479.805,
|
||
|
|
"logps/rejected": -400.1239798964968,
|
||
|
|
"loss": 0.5051,
|
||
|
|
"loss/base_kto": 3.418736696243286,
|
||
|
|
"loss/total_with_kl": 4.0411505699157715,
|
||
|
|
"metrics/advantage_var": 373.8221740722656,
|
||
|
|
"regularization/advantage_var": 373.8221740722656,
|
||
|
|
"regularization/kl": 0.6224138736724854,
|
||
|
|
"rewards/chosen": 0.2725811063326322,
|
||
|
|
"rewards/margins": 0.5847772183154273,
|
||
|
|
"rewards/rejected": -0.31219611198279507,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 12.080039978027344,
|
||
|
|
"kl": 15.027986526489258,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -38061731.26934984,
|
||
|
|
"logits/rejected": -37633844.49211357,
|
||
|
|
"logps/chosen": -465.4192143962848,
|
||
|
|
"logps/rejected": -385.28916108044166,
|
||
|
|
"loss": 0.5599,
|
||
|
|
"loss/base_kto": 3.164376735687256,
|
||
|
|
"loss/total_with_kl": 4.479323863983154,
|
||
|
|
"metrics/advantage_var": 789.7579956054688,
|
||
|
|
"regularization/advantage_var": 789.7579956054688,
|
||
|
|
"regularization/kl": 1.3149470090866089,
|
||
|
|
"rewards/chosen": 0.5711726610874613,
|
||
|
|
"rewards/margins": 0.9308842690114805,
|
||
|
|
"rewards/rejected": -0.35971160792401913,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 26.646738052368164,
|
||
|
|
"kl": 12.82618236541748,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -36088989.79672131,
|
||
|
|
"logits/rejected": -37796316.84776119,
|
||
|
|
"logps/chosen": -490.16296106557377,
|
||
|
|
"logps/rejected": -369.60452425373137,
|
||
|
|
"loss": 0.5234,
|
||
|
|
"loss/base_kto": 3.207984209060669,
|
||
|
|
"loss/total_with_kl": 4.187113285064697,
|
||
|
|
"metrics/advantage_var": 588.0658569335938,
|
||
|
|
"regularization/advantage_var": 588.0658569335938,
|
||
|
|
"regularization/kl": 0.9791294932365417,
|
||
|
|
"rewards/chosen": 0.44803656906378075,
|
||
|
|
"rewards/margins": 0.8352686065083936,
|
||
|
|
"rewards/rejected": -0.3872320374446129,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 24.08795738220215,
|
||
|
|
"kl": 11.3661470413208,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -36696518.02232855,
|
||
|
|
"logits/rejected": -37635173.92955589,
|
||
|
|
"logps/chosen": -468.5043361244019,
|
||
|
|
"logps/rejected": -355.8814127105666,
|
||
|
|
"loss": 0.5215,
|
||
|
|
"loss/base_kto": 3.134793519973755,
|
||
|
|
"loss/total_with_kl": 4.172141075134277,
|
||
|
|
"metrics/advantage_var": 623.0315551757812,
|
||
|
|
"regularization/advantage_var": 623.0315551757812,
|
||
|
|
"regularization/kl": 1.037347435951233,
|
||
|
|
"rewards/chosen": 0.4973068602347488,
|
||
|
|
"rewards/margins": 0.9334474493427121,
|
||
|
|
"rewards/rejected": -0.4361405891079633,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 9.80352783203125,
|
||
|
|
"kl": 7.617015838623047,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -34527132.90322581,
|
||
|
|
"logits/rejected": -35678018.71515152,
|
||
|
|
"logps/chosen": -494.403377016129,
|
||
|
|
"logps/rejected": -370.2385890151515,
|
||
|
|
"loss": 0.5081,
|
||
|
|
"loss/base_kto": 3.1501028537750244,
|
||
|
|
"loss/total_with_kl": 4.064969539642334,
|
||
|
|
"metrics/advantage_var": 549.4691772460938,
|
||
|
|
"regularization/advantage_var": 549.4691772460938,
|
||
|
|
"regularization/kl": 0.9148662686347961,
|
||
|
|
"rewards/chosen": 0.4089845718876008,
|
||
|
|
"rewards/margins": 0.9288768540845704,
|
||
|
|
"rewards/rejected": -0.5198922821969697,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 29.720600128173828,
|
||
|
|
"kl": 8.63927936553955,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -35373025.06344411,
|
||
|
|
"logits/rejected": -36494829.77346278,
|
||
|
|
"logps/chosen": -491.7433912386707,
|
||
|
|
"logps/rejected": -386.94993932038835,
|
||
|
|
"loss": 0.5123,
|
||
|
|
"loss/base_kto": 3.148014783859253,
|
||
|
|
"loss/total_with_kl": 4.098679065704346,
|
||
|
|
"metrics/advantage_var": 570.9694213867188,
|
||
|
|
"regularization/advantage_var": 570.9694213867188,
|
||
|
|
"regularization/kl": 0.9506641626358032,
|
||
|
|
"rewards/chosen": 0.5039736007275538,
|
||
|
|
"rewards/margins": 0.9435728446817154,
|
||
|
|
"rewards/rejected": -0.4395992439541616,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 29.414081573486328,
|
||
|
|
"kl": 20.815481185913086,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -35595772.83950617,
|
||
|
|
"logits/rejected": -36601716.658227846,
|
||
|
|
"logps/chosen": -471.42158564814815,
|
||
|
|
"logps/rejected": -394.53926028481015,
|
||
|
|
"loss": 0.5574,
|
||
|
|
"loss/base_kto": 3.1448068618774414,
|
||
|
|
"loss/total_with_kl": 4.458906650543213,
|
||
|
|
"metrics/advantage_var": 789.2491455078125,
|
||
|
|
"regularization/advantage_var": 789.2491455078125,
|
||
|
|
"regularization/kl": 1.314099669456482,
|
||
|
|
"rewards/chosen": 0.6382946909209828,
|
||
|
|
"rewards/margins": 0.9447758712327411,
|
||
|
|
"rewards/rejected": -0.3064811803117583,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 26.01615333557129,
|
||
|
|
"kl": 15.914934158325195,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -34975544.97805642,
|
||
|
|
"logits/rejected": -36403914.566978194,
|
||
|
|
"logps/chosen": -472.46581112852664,
|
||
|
|
"logps/rejected": -408.1698062694704,
|
||
|
|
"loss": 0.5317,
|
||
|
|
"loss/base_kto": 3.1391713619232178,
|
||
|
|
"loss/total_with_kl": 4.253345012664795,
|
||
|
|
"metrics/advantage_var": 669.1731567382812,
|
||
|
|
"regularization/advantage_var": 669.1731567382812,
|
||
|
|
"regularization/kl": 1.1141732931137085,
|
||
|
|
"rewards/chosen": 0.5612045814251078,
|
||
|
|
"rewards/margins": 0.9698403715280575,
|
||
|
|
"rewards/rejected": -0.40863579010294976,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 17.740093231201172,
|
||
|
|
"kl": 21.399534225463867,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35322617.6,
|
||
|
|
"logits/rejected": -37008326.4,
|
||
|
|
"logps/chosen": -469.24599609375,
|
||
|
|
"logps/rejected": -384.849755859375,
|
||
|
|
"loss": 0.5626,
|
||
|
|
"loss/base_kto": 3.1882617473602295,
|
||
|
|
"loss/total_with_kl": 4.500464916229248,
|
||
|
|
"metrics/advantage_var": 788.1102294921875,
|
||
|
|
"regularization/advantage_var": 788.1102294921875,
|
||
|
|
"regularization/kl": 1.3122034072875977,
|
||
|
|
"rewards/chosen": 0.6511719703674317,
|
||
|
|
"rewards/margins": 0.9023769855499268,
|
||
|
|
"rewards/rejected": -0.2512050151824951,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 22.442768096923828,
|
||
|
|
"kl": 12.02442455291748,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -36062712.0743034,
|
||
|
|
"logits/rejected": -36736658.977917984,
|
||
|
|
"logps/chosen": -479.61977554179566,
|
||
|
|
"logps/rejected": -369.4289974369085,
|
||
|
|
"loss": 0.5109,
|
||
|
|
"loss/base_kto": 3.197706937789917,
|
||
|
|
"loss/total_with_kl": 4.087076663970947,
|
||
|
|
"metrics/advantage_var": 534.15576171875,
|
||
|
|
"regularization/advantage_var": 534.15576171875,
|
||
|
|
"regularization/kl": 0.8893694281578064,
|
||
|
|
"rewards/chosen": 0.47495215448432665,
|
||
|
|
"rewards/margins": 0.8866932006076508,
|
||
|
|
"rewards/rejected": -0.41174104612332413,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 12.254045486450195,
|
||
|
|
"kl": 25.559003829956055,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -37180858.98432602,
|
||
|
|
"logits/rejected": -36697757.906542055,
|
||
|
|
"logps/chosen": -469.63665752351096,
|
||
|
|
"logps/rejected": -371.635343652648,
|
||
|
|
"loss": 0.5212,
|
||
|
|
"loss/base_kto": 3.090243101119995,
|
||
|
|
"loss/total_with_kl": 4.169286727905273,
|
||
|
|
"metrics/advantage_var": 648.07421875,
|
||
|
|
"regularization/advantage_var": 648.07421875,
|
||
|
|
"regularization/kl": 1.0790436267852783,
|
||
|
|
"rewards/chosen": 0.8071287149172218,
|
||
|
|
"rewards/margins": 0.9585992438733647,
|
||
|
|
"rewards/rejected": -0.1514705289561429,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 19.815689086914062,
|
||
|
|
"kl": 14.280487060546875,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34838058.27522936,
|
||
|
|
"logits/rejected": -34883070.36421725,
|
||
|
|
"logps/chosen": -481.26739296636083,
|
||
|
|
"logps/rejected": -363.7084165335463,
|
||
|
|
"loss": 0.5205,
|
||
|
|
"loss/base_kto": 3.2393035888671875,
|
||
|
|
"loss/total_with_kl": 4.163845062255859,
|
||
|
|
"metrics/advantage_var": 555.2802124023438,
|
||
|
|
"regularization/advantage_var": 555.2802124023438,
|
||
|
|
"regularization/kl": 0.9245415925979614,
|
||
|
|
"rewards/chosen": 0.544169901343296,
|
||
|
|
"rewards/margins": 0.8161835076981283,
|
||
|
|
"rewards/rejected": -0.2720136063548323,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 19.823097229003906,
|
||
|
|
"kl": 21.81817054748535,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -36044031.17152104,
|
||
|
|
"logits/rejected": -38181488.918429,
|
||
|
|
"logps/chosen": -485.4643001618123,
|
||
|
|
"logps/rejected": -362.6658327039275,
|
||
|
|
"loss": 0.5133,
|
||
|
|
"loss/base_kto": 3.191328763961792,
|
||
|
|
"loss/total_with_kl": 4.106320858001709,
|
||
|
|
"metrics/advantage_var": 549.5450439453125,
|
||
|
|
"regularization/advantage_var": 549.5450439453125,
|
||
|
|
"regularization/kl": 0.9149925112724304,
|
||
|
|
"rewards/chosen": 0.6180900894708232,
|
||
|
|
"rewards/margins": 0.8244822220945015,
|
||
|
|
"rewards/rejected": -0.20639213262367825,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 8.847859382629395,
|
||
|
|
"kl": 13.577710151672363,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -35605410.90909091,
|
||
|
|
"logits/rejected": -37161368.323987536,
|
||
|
|
"logps/chosen": -480.18221003134795,
|
||
|
|
"logps/rejected": -396.22765771028037,
|
||
|
|
"loss": 0.5425,
|
||
|
|
"loss/base_kto": 3.2130470275878906,
|
||
|
|
"loss/total_with_kl": 4.340249538421631,
|
||
|
|
"metrics/advantage_var": 676.9984130859375,
|
||
|
|
"regularization/advantage_var": 676.9984130859375,
|
||
|
|
"regularization/kl": 1.1272023916244507,
|
||
|
|
"rewards/chosen": 0.47794239199647337,
|
||
|
|
"rewards/margins": 0.831569843480179,
|
||
|
|
"rewards/rejected": -0.3536274514837057,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 10.016629219055176,
|
||
|
|
"kl": 14.0739164352417,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -36337693.3036566,
|
||
|
|
"logits/rejected": -37938152.40552995,
|
||
|
|
"logps/chosen": -474.0241454689984,
|
||
|
|
"logps/rejected": -374.08172523041475,
|
||
|
|
"loss": 0.524,
|
||
|
|
"loss/base_kto": 3.190122365951538,
|
||
|
|
"loss/total_with_kl": 4.191695690155029,
|
||
|
|
"metrics/advantage_var": 601.5457763671875,
|
||
|
|
"regularization/advantage_var": 601.5457763671875,
|
||
|
|
"regularization/kl": 1.0015738010406494,
|
||
|
|
"rewards/chosen": 0.5609835820281449,
|
||
|
|
"rewards/margins": 0.8730009261381679,
|
||
|
|
"rewards/rejected": -0.312017344110023,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 26.99571418762207,
|
||
|
|
"kl": 14.935075759887695,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -35748702.81481481,
|
||
|
|
"logits/rejected": -36598557.164556965,
|
||
|
|
"logps/chosen": -491.79050925925924,
|
||
|
|
"logps/rejected": -366.63815268987344,
|
||
|
|
"loss": 0.5187,
|
||
|
|
"loss/base_kto": 3.191035032272339,
|
||
|
|
"loss/total_with_kl": 4.149347305297852,
|
||
|
|
"metrics/advantage_var": 575.5628051757812,
|
||
|
|
"regularization/advantage_var": 575.5628051757812,
|
||
|
|
"regularization/kl": 0.9583120346069336,
|
||
|
|
"rewards/chosen": 0.5541232073748553,
|
||
|
|
"rewards/margins": 0.8421052788771546,
|
||
|
|
"rewards/rejected": -0.28798207150229926,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 25.332813262939453,
|
||
|
|
"kl": 19.600326538085938,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -35614053.78313253,
|
||
|
|
"logits/rejected": -37066319.79220779,
|
||
|
|
"logps/chosen": -474.968938253012,
|
||
|
|
"logps/rejected": -362.3044592126623,
|
||
|
|
"loss": 0.5097,
|
||
|
|
"loss/base_kto": 3.144695997238159,
|
||
|
|
"loss/total_with_kl": 4.077559947967529,
|
||
|
|
"metrics/advantage_var": 560.2787475585938,
|
||
|
|
"regularization/advantage_var": 560.2787475585938,
|
||
|
|
"regularization/kl": 0.9328641891479492,
|
||
|
|
"rewards/chosen": 0.6686081484139684,
|
||
|
|
"rewards/margins": 0.910741945403112,
|
||
|
|
"rewards/rejected": -0.24213379698914367,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 15.155717849731445,
|
||
|
|
"kl": 20.27742576599121,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -34903524.07272727,
|
||
|
|
"logits/rejected": -36508767.79354839,
|
||
|
|
"logps/chosen": -472.25364583333334,
|
||
|
|
"logps/rejected": -385.275,
|
||
|
|
"loss": 0.4985,
|
||
|
|
"loss/base_kto": 3.2158966064453125,
|
||
|
|
"loss/total_with_kl": 3.988041400909424,
|
||
|
|
"metrics/advantage_var": 463.75030517578125,
|
||
|
|
"regularization/advantage_var": 463.75030517578125,
|
||
|
|
"regularization/kl": 0.7721441984176636,
|
||
|
|
"rewards/chosen": 0.6540827433268229,
|
||
|
|
"rewards/margins": 0.8281449676841818,
|
||
|
|
"rewards/rejected": -0.17406222435735888,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 21.98465919494629,
|
||
|
|
"kl": 21.62624168395996,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -35109864.95176849,
|
||
|
|
"logits/rejected": -36197684.1337386,
|
||
|
|
"logps/chosen": -491.34405144694534,
|
||
|
|
"logps/rejected": -357.59213525835867,
|
||
|
|
"loss": 0.4973,
|
||
|
|
"loss/base_kto": 3.1464684009552,
|
||
|
|
"loss/total_with_kl": 3.97809100151062,
|
||
|
|
"metrics/advantage_var": 499.4730529785156,
|
||
|
|
"regularization/advantage_var": 499.4730529785156,
|
||
|
|
"regularization/kl": 0.8316227197647095,
|
||
|
|
"rewards/chosen": 0.6417734323973824,
|
||
|
|
"rewards/margins": 0.8699830004094692,
|
||
|
|
"rewards/rejected": -0.2282095680120868,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 16.958070755004883,
|
||
|
|
"kl": 9.069159507751465,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -34802558.979266346,
|
||
|
|
"logits/rejected": -37141838.01531394,
|
||
|
|
"logps/chosen": -492.8044258373206,
|
||
|
|
"logps/rejected": -385.7053024502297,
|
||
|
|
"loss": 0.5338,
|
||
|
|
"loss/base_kto": 3.22055983543396,
|
||
|
|
"loss/total_with_kl": 4.270147323608398,
|
||
|
|
"metrics/advantage_var": 630.3829345703125,
|
||
|
|
"regularization/advantage_var": 630.3829345703125,
|
||
|
|
"regularization/kl": 1.0495874881744385,
|
||
|
|
"rewards/chosen": 0.3879310814767743,
|
||
|
|
"rewards/margins": 0.8503653915720079,
|
||
|
|
"rewards/rejected": -0.46243431009523356,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 16.89603614807129,
|
||
|
|
"kl": 12.140111923217773,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34872773.04242424,
|
||
|
|
"logits/rejected": -37482796.59354839,
|
||
|
|
"logps/chosen": -496.0032196969697,
|
||
|
|
"logps/rejected": -378.95584677419356,
|
||
|
|
"loss": 0.533,
|
||
|
|
"loss/base_kto": 3.2100415229797363,
|
||
|
|
"loss/total_with_kl": 4.264197826385498,
|
||
|
|
"metrics/advantage_var": 633.1267700195312,
|
||
|
|
"regularization/advantage_var": 633.1267700195312,
|
||
|
|
"regularization/kl": 1.0541560649871826,
|
||
|
|
"rewards/chosen": 0.536898202607126,
|
||
|
|
"rewards/margins": 0.8187301039113094,
|
||
|
|
"rewards/rejected": -0.28183190130418345,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 16.812923431396484,
|
||
|
|
"kl": 12.90421199798584,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -34195034.546623796,
|
||
|
|
"logits/rejected": -37324971.185410336,
|
||
|
|
"logps/chosen": -483.4717142282958,
|
||
|
|
"logps/rejected": -359.94849449088144,
|
||
|
|
"loss": 0.5093,
|
||
|
|
"loss/base_kto": 3.219403028488159,
|
||
|
|
"loss/total_with_kl": 4.074177265167236,
|
||
|
|
"metrics/advantage_var": 513.3780517578125,
|
||
|
|
"regularization/advantage_var": 513.3780517578125,
|
||
|
|
"regularization/kl": 0.8547744750976562,
|
||
|
|
"rewards/chosen": 0.5112144249428506,
|
||
|
|
"rewards/margins": 0.8330791472904,
|
||
|
|
"rewards/rejected": -0.32186472234754937,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 14.058871269226074,
|
||
|
|
"kl": 14.516571044921875,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -34791809.80847724,
|
||
|
|
"logits/rejected": -35851013.972006224,
|
||
|
|
"logps/chosen": -458.04017857142856,
|
||
|
|
"logps/rejected": -373.04199066874025,
|
||
|
|
"loss": 0.5007,
|
||
|
|
"loss/base_kto": 3.2256600856781006,
|
||
|
|
"loss/total_with_kl": 4.005448818206787,
|
||
|
|
"metrics/advantage_var": 468.3414001464844,
|
||
|
|
"regularization/advantage_var": 468.3414001464844,
|
||
|
|
"regularization/kl": 0.7797884941101074,
|
||
|
|
"rewards/chosen": 0.474783377894427,
|
||
|
|
"rewards/margins": 0.8102318372645669,
|
||
|
|
"rewards/rejected": -0.33544845937013995,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 11.235255241394043,
|
||
|
|
"kl": 17.65205955505371,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -34895403.256528415,
|
||
|
|
"logits/rejected": -35339379.58664547,
|
||
|
|
"logps/chosen": -483.04881912442397,
|
||
|
|
"logps/rejected": -374.9543918918919,
|
||
|
|
"loss": 0.5281,
|
||
|
|
"loss/base_kto": 3.202352285385132,
|
||
|
|
"loss/total_with_kl": 4.224853038787842,
|
||
|
|
"metrics/advantage_var": 614.1143798828125,
|
||
|
|
"regularization/advantage_var": 614.1143798828125,
|
||
|
|
"regularization/kl": 1.0225003957748413,
|
||
|
|
"rewards/chosen": 0.6097218503234207,
|
||
|
|
"rewards/margins": 0.8648844638934476,
|
||
|
|
"rewards/rejected": -0.25516261357002684,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 16.98574447631836,
|
||
|
|
"kl": 9.064486503601074,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -36055082.1933439,
|
||
|
|
"logits/rejected": -36566803.328197226,
|
||
|
|
"logps/chosen": -484.90664619651346,
|
||
|
|
"logps/rejected": -381.66838405238826,
|
||
|
|
"loss": 0.5263,
|
||
|
|
"loss/base_kto": 3.227729558944702,
|
||
|
|
"loss/total_with_kl": 4.210367202758789,
|
||
|
|
"metrics/advantage_var": 590.1730346679688,
|
||
|
|
"regularization/advantage_var": 590.1730346679688,
|
||
|
|
"regularization/kl": 0.9826381802558899,
|
||
|
|
"rewards/chosen": 0.41754561483198793,
|
||
|
|
"rewards/margins": 0.8201956691523365,
|
||
|
|
"rewards/rejected": -0.4026500543203486,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 20.735435485839844,
|
||
|
|
"kl": 13.433310508728027,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -34812483.2,
|
||
|
|
"logits/rejected": -35821484.8,
|
||
|
|
"logps/chosen": -489.5673828125,
|
||
|
|
"logps/rejected": -380.184228515625,
|
||
|
|
"loss": 0.5075,
|
||
|
|
"loss/base_kto": 3.173840045928955,
|
||
|
|
"loss/total_with_kl": 4.0600409507751465,
|
||
|
|
"metrics/advantage_var": 532.2528076171875,
|
||
|
|
"regularization/advantage_var": 532.2528076171875,
|
||
|
|
"regularization/kl": 0.8862008452415466,
|
||
|
|
"rewards/chosen": 0.5355085372924805,
|
||
|
|
"rewards/margins": 0.8867841720581056,
|
||
|
|
"rewards/rejected": -0.351275634765625,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 10.443603515625,
|
||
|
|
"kl": 15.014068603515625,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -35359488.76876877,
|
||
|
|
"logits/rejected": -36065006.488599345,
|
||
|
|
"logps/chosen": -468.1949605855856,
|
||
|
|
"logps/rejected": -372.7525447882736,
|
||
|
|
"loss": 0.5377,
|
||
|
|
"loss/base_kto": 3.2357544898986816,
|
||
|
|
"loss/total_with_kl": 4.301290512084961,
|
||
|
|
"metrics/advantage_var": 639.9616088867188,
|
||
|
|
"regularization/advantage_var": 639.9616088867188,
|
||
|
|
"regularization/kl": 1.0655361413955688,
|
||
|
|
"rewards/chosen": 0.5401951328770176,
|
||
|
|
"rewards/margins": 0.8020933600343618,
|
||
|
|
"rewards/rejected": -0.26189822715734423,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 10.77260684967041,
|
||
|
|
"kl": 20.780960083007812,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -35667223.27272727,
|
||
|
|
"logits/rejected": -38056867.489096574,
|
||
|
|
"logps/chosen": -479.54545454545456,
|
||
|
|
"logps/rejected": -379.64751265576325,
|
||
|
|
"loss": 0.5154,
|
||
|
|
"loss/base_kto": 3.1758131980895996,
|
||
|
|
"loss/total_with_kl": 4.122966766357422,
|
||
|
|
"metrics/advantage_var": 568.861083984375,
|
||
|
|
"regularization/advantage_var": 568.861083984375,
|
||
|
|
"regularization/kl": 0.9471536874771118,
|
||
|
|
"rewards/chosen": 0.6971722067710374,
|
||
|
|
"rewards/margins": 0.8432453312614604,
|
||
|
|
"rewards/rejected": -0.146073124490423,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 21.820354461669922,
|
||
|
|
"kl": 18.302148818969727,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -34120516.2403698,
|
||
|
|
"logits/rejected": -36425189.22345483,
|
||
|
|
"logps/chosen": -480.7200500770416,
|
||
|
|
"logps/rejected": -358.38076961172743,
|
||
|
|
"loss": 0.4989,
|
||
|
|
"loss/base_kto": 3.1738641262054443,
|
||
|
|
"loss/total_with_kl": 3.991513967514038,
|
||
|
|
"metrics/advantage_var": 491.0809020996094,
|
||
|
|
"regularization/advantage_var": 491.0809020996094,
|
||
|
|
"regularization/kl": 0.8176496624946594,
|
||
|
|
"rewards/chosen": 0.5981124619306144,
|
||
|
|
"rewards/margins": 0.8380587577640509,
|
||
|
|
"rewards/rejected": -0.23994629583343652,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 19.11968231201172,
|
||
|
|
"kl": 15.336816787719727,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -35227818.131661445,
|
||
|
|
"logits/rejected": -35232662.728971966,
|
||
|
|
"logps/chosen": -469.18172021943576,
|
||
|
|
"logps/rejected": -366.48568925233644,
|
||
|
|
"loss": 0.5098,
|
||
|
|
"loss/base_kto": 3.1704258918762207,
|
||
|
|
"loss/total_with_kl": 4.078118801116943,
|
||
|
|
"metrics/advantage_var": 545.1610107421875,
|
||
|
|
"regularization/advantage_var": 545.1610107421875,
|
||
|
|
"regularization/kl": 0.9076930284500122,
|
||
|
|
"rewards/chosen": 0.601460567462407,
|
||
|
|
"rewards/margins": 0.8971618528513892,
|
||
|
|
"rewards/rejected": -0.29570128538898216,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 15.077584266662598,
|
||
|
|
"kl": 14.563433647155762,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -35686443.150729336,
|
||
|
|
"logits/rejected": -36516939.680241324,
|
||
|
|
"logps/chosen": -496.18096636953,
|
||
|
|
"logps/rejected": -382.3719362745098,
|
||
|
|
"loss": 0.5261,
|
||
|
|
"loss/base_kto": 3.1742496490478516,
|
||
|
|
"loss/total_with_kl": 4.2089457511901855,
|
||
|
|
"metrics/advantage_var": 621.4389038085938,
|
||
|
|
"regularization/advantage_var": 621.4389038085938,
|
||
|
|
"regularization/kl": 1.0346958637237549,
|
||
|
|
"rewards/chosen": 0.5384568967154578,
|
||
|
|
"rewards/margins": 0.863727513525884,
|
||
|
|
"rewards/rejected": -0.32527061681042607,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 17.804264068603516,
|
||
|
|
"kl": 12.497262954711914,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -36677968.4338118,
|
||
|
|
"logits/rejected": -36208094.284839205,
|
||
|
|
"logps/chosen": -500.66487240829343,
|
||
|
|
"logps/rejected": -386.2694056278714,
|
||
|
|
"loss": 0.51,
|
||
|
|
"loss/base_kto": 3.1934421062469482,
|
||
|
|
"loss/total_with_kl": 4.079799652099609,
|
||
|
|
"metrics/advantage_var": 532.346923828125,
|
||
|
|
"regularization/advantage_var": 532.346923828125,
|
||
|
|
"regularization/kl": 0.8863576054573059,
|
||
|
|
"rewards/chosen": 0.502348295999676,
|
||
|
|
"rewards/margins": 0.8429343288378986,
|
||
|
|
"rewards/rejected": -0.34058603283822264,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 16.499343872070312,
|
||
|
|
"kl": 12.454108238220215,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -34069816.756589144,
|
||
|
|
"logits/rejected": -35322900.963779524,
|
||
|
|
"logps/chosen": -476.69578488372093,
|
||
|
|
"logps/rejected": -396.577312992126,
|
||
|
|
"loss": 0.5123,
|
||
|
|
"loss/base_kto": 3.1449525356292725,
|
||
|
|
"loss/total_with_kl": 4.098038673400879,
|
||
|
|
"metrics/advantage_var": 572.4239501953125,
|
||
|
|
"regularization/advantage_var": 572.4239501953125,
|
||
|
|
"regularization/kl": 0.953085720539093,
|
||
|
|
"rewards/chosen": 0.44505615234375,
|
||
|
|
"rewards/margins": 0.9144545667753445,
|
||
|
|
"rewards/rejected": -0.46939841443159447,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 68.79667663574219,
|
||
|
|
"kl": 19.708200454711914,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -34935476.28169014,
|
||
|
|
"logits/rejected": -34988153.41029641,
|
||
|
|
"logps/chosen": -475.5610817683881,
|
||
|
|
"logps/rejected": -365.95154056162244,
|
||
|
|
"loss": 0.497,
|
||
|
|
"loss/base_kto": 3.1570169925689697,
|
||
|
|
"loss/total_with_kl": 3.9757080078125,
|
||
|
|
"metrics/advantage_var": 491.70635986328125,
|
||
|
|
"regularization/advantage_var": 491.70635986328125,
|
||
|
|
"regularization/kl": 0.818691074848175,
|
||
|
|
"rewards/chosen": 0.6771548753239925,
|
||
|
|
"rewards/margins": 0.8825103246493875,
|
||
|
|
"rewards/rejected": -0.2053554493253949,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 11.845552444458008,
|
||
|
|
"kl": 17.667652130126953,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -35823884.8,
|
||
|
|
"logits/rejected": -35276588.8,
|
||
|
|
"logps/chosen": -486.55625,
|
||
|
|
"logps/rejected": -366.40927734375,
|
||
|
|
"loss": 0.5097,
|
||
|
|
"loss/base_kto": 3.158928394317627,
|
||
|
|
"loss/total_with_kl": 4.077810287475586,
|
||
|
|
"metrics/advantage_var": 551.8809814453125,
|
||
|
|
"regularization/advantage_var": 551.8809814453125,
|
||
|
|
"regularization/kl": 0.9188818335533142,
|
||
|
|
"rewards/chosen": 0.6351908683776856,
|
||
|
|
"rewards/margins": 0.8914699792861939,
|
||
|
|
"rewards/rejected": -0.2562791109085083,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 18.535634994506836,
|
||
|
|
"kl": 16.133092880249023,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -35617760.19875777,
|
||
|
|
"logits/rejected": -35217517.48427673,
|
||
|
|
"logps/chosen": -473.39091614906835,
|
||
|
|
"logps/rejected": -381.6313138757862,
|
||
|
|
"loss": 0.5112,
|
||
|
|
"loss/base_kto": 3.206963300704956,
|
||
|
|
"loss/total_with_kl": 4.089570045471191,
|
||
|
|
"metrics/advantage_var": 530.0939331054688,
|
||
|
|
"regularization/advantage_var": 530.0939331054688,
|
||
|
|
"regularization/kl": 0.8826065063476562,
|
||
|
|
"rewards/chosen": 0.613495015209506,
|
||
|
|
"rewards/margins": 0.8020739387528003,
|
||
|
|
"rewards/rejected": -0.18857892354329428,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 11.974020957946777,
|
||
|
|
"kl": 16.73115348815918,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -33763296.49230769,
|
||
|
|
"logits/rejected": -34868994.43809524,
|
||
|
|
"logps/chosen": -494.2183173076923,
|
||
|
|
"logps/rejected": -364.3859623015873,
|
||
|
|
"loss": 0.5122,
|
||
|
|
"loss/base_kto": 3.1801657676696777,
|
||
|
|
"loss/total_with_kl": 4.097968578338623,
|
||
|
|
"metrics/advantage_var": 551.2330322265625,
|
||
|
|
"regularization/advantage_var": 551.2330322265625,
|
||
|
|
"regularization/kl": 0.9178029894828796,
|
||
|
|
"rewards/chosen": 0.6548098520132212,
|
||
|
|
"rewards/margins": 0.8398223787524325,
|
||
|
|
"rewards/rejected": -0.1850125267392113,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 9.843354225158691,
|
||
|
|
"kl": 14.801050186157227,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -36169348.49840256,
|
||
|
|
"logits/rejected": -36987512.56269113,
|
||
|
|
"logps/chosen": -480.01472643769966,
|
||
|
|
"logps/rejected": -367.8846999235474,
|
||
|
|
"loss": 0.5033,
|
||
|
|
"loss/base_kto": 3.2000465393066406,
|
||
|
|
"loss/total_with_kl": 4.026675224304199,
|
||
|
|
"metrics/advantage_var": 496.4736022949219,
|
||
|
|
"regularization/advantage_var": 496.4736022949219,
|
||
|
|
"regularization/kl": 0.8266285061836243,
|
||
|
|
"rewards/chosen": 0.4919905982459315,
|
||
|
|
"rewards/margins": 0.8278221966502821,
|
||
|
|
"rewards/rejected": -0.3358315984043506,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 13.826387405395508,
|
||
|
|
"kl": 13.46362590789795,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -34888960.39813375,
|
||
|
|
"logits/rejected": -35481497.1177394,
|
||
|
|
"logps/chosen": -495.83466174183513,
|
||
|
|
"logps/rejected": -372.1906887755102,
|
||
|
|
"loss": 0.5309,
|
||
|
|
"loss/base_kto": 3.240717649459839,
|
||
|
|
"loss/total_with_kl": 4.24738883972168,
|
||
|
|
"metrics/advantage_var": 604.6072998046875,
|
||
|
|
"regularization/advantage_var": 604.6072998046875,
|
||
|
|
"regularization/kl": 1.0066711902618408,
|
||
|
|
"rewards/chosen": 0.5279172716392885,
|
||
|
|
"rewards/margins": 0.7773599545492914,
|
||
|
|
"rewards/rejected": -0.24944268291000293,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 10.795230865478516,
|
||
|
|
"kl": 17.875226974487305,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34820278.46390169,
|
||
|
|
"logits/rejected": -36204207.5661882,
|
||
|
|
"logps/chosen": -492.9363959293395,
|
||
|
|
"logps/rejected": -363.6246012759171,
|
||
|
|
"loss": 0.4693,
|
||
|
|
"loss/base_kto": 3.107238292694092,
|
||
|
|
"loss/total_with_kl": 3.754481554031372,
|
||
|
|
"metrics/advantage_var": 388.7345886230469,
|
||
|
|
"regularization/advantage_var": 388.7345886230469,
|
||
|
|
"regularization/kl": 0.6472430229187012,
|
||
|
|
"rewards/chosen": 0.6626991002424155,
|
||
|
|
"rewards/margins": 0.9209322025571958,
|
||
|
|
"rewards/rejected": -0.2582331023147802,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 19.27705955505371,
|
||
|
|
"kl": 18.832422256469727,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34594490.97072419,
|
||
|
|
"logits/rejected": -34871091.524564184,
|
||
|
|
"logps/chosen": -475.6154661016949,
|
||
|
|
"logps/rejected": -404.0041848256735,
|
||
|
|
"loss": 0.4666,
|
||
|
|
"loss/base_kto": 3.088207721710205,
|
||
|
|
"loss/total_with_kl": 3.73294997215271,
|
||
|
|
"metrics/advantage_var": 387.2326354980469,
|
||
|
|
"regularization/advantage_var": 387.2326354980469,
|
||
|
|
"regularization/kl": 0.6447422504425049,
|
||
|
|
"rewards/chosen": 0.660996071179218,
|
||
|
|
"rewards/margins": 0.9134467149211702,
|
||
|
|
"rewards/rejected": -0.25245064374195225,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 16.828039169311523,
|
||
|
|
"kl": 18.012939453125,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -33559192.89433385,
|
||
|
|
"logits/rejected": -35118578.11802233,
|
||
|
|
"logps/chosen": -491.3185777182236,
|
||
|
|
"logps/rejected": -388.03668261562996,
|
||
|
|
"loss": 0.4702,
|
||
|
|
"loss/base_kto": 3.0965676307678223,
|
||
|
|
"loss/total_with_kl": 3.761559247970581,
|
||
|
|
"metrics/advantage_var": 399.39447021484375,
|
||
|
|
"regularization/advantage_var": 399.39447021484375,
|
||
|
|
"regularization/kl": 0.6649917960166931,
|
||
|
|
"rewards/chosen": 0.6639900616445731,
|
||
|
|
"rewards/margins": 0.9009370072000455,
|
||
|
|
"rewards/rejected": -0.2369469455554725,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 13.841697692871094,
|
||
|
|
"kl": 18.363540649414062,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -36010543.16917923,
|
||
|
|
"logits/rejected": -36651898.56515373,
|
||
|
|
"logps/chosen": -480.6165201005025,
|
||
|
|
"logps/rejected": -403.83848828696927,
|
||
|
|
"loss": 0.4719,
|
||
|
|
"loss/base_kto": 3.1108250617980957,
|
||
|
|
"loss/total_with_kl": 3.7754433155059814,
|
||
|
|
"metrics/advantage_var": 399.1703796386719,
|
||
|
|
"regularization/advantage_var": 399.1703796386719,
|
||
|
|
"regularization/kl": 0.6646186709403992,
|
||
|
|
"rewards/chosen": 0.6045121236063128,
|
||
|
|
"rewards/margins": 0.9177538169270829,
|
||
|
|
"rewards/rejected": -0.31324169332077006,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 13.886054039001465,
|
||
|
|
"kl": 10.77209758758545,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -33384008.681388013,
|
||
|
|
"logits/rejected": -35382937.758513935,
|
||
|
|
"logps/chosen": -479.59261632492115,
|
||
|
|
"logps/rejected": -364.9218024380805,
|
||
|
|
"loss": 0.4606,
|
||
|
|
"loss/base_kto": 3.1307339668273926,
|
||
|
|
"loss/total_with_kl": 3.6846272945404053,
|
||
|
|
"metrics/advantage_var": 332.6686706542969,
|
||
|
|
"regularization/advantage_var": 332.6686706542969,
|
||
|
|
"regularization/kl": 0.5538933277130127,
|
||
|
|
"rewards/chosen": 0.4953169476722693,
|
||
|
|
"rewards/margins": 0.8828692149299124,
|
||
|
|
"rewards/rejected": -0.3875522672576432,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 12.666690826416016,
|
||
|
|
"kl": 13.711188316345215,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -35280860.52283464,
|
||
|
|
"logits/rejected": -34790204.72558139,
|
||
|
|
"logps/chosen": -479.02677165354334,
|
||
|
|
"logps/rejected": -378.6731346899225,
|
||
|
|
"loss": 0.4647,
|
||
|
|
"loss/base_kto": 3.136176824569702,
|
||
|
|
"loss/total_with_kl": 3.7178847789764404,
|
||
|
|
"metrics/advantage_var": 349.37420654296875,
|
||
|
|
"regularization/advantage_var": 349.37420654296875,
|
||
|
|
"regularization/kl": 0.5817080736160278,
|
||
|
|
"rewards/chosen": 0.5767086960199311,
|
||
|
|
"rewards/margins": 0.8669471636873851,
|
||
|
|
"rewards/rejected": -0.290238467667454,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 8.175899505615234,
|
||
|
|
"kl": 13.4633207321167,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -35221138.61469265,
|
||
|
|
"logits/rejected": -36860077.729200654,
|
||
|
|
"logps/chosen": -483.1257496251874,
|
||
|
|
"logps/rejected": -372.60741231647637,
|
||
|
|
"loss": 0.4736,
|
||
|
|
"loss/base_kto": 3.147956132888794,
|
||
|
|
"loss/total_with_kl": 3.7891335487365723,
|
||
|
|
"metrics/advantage_var": 385.0915222167969,
|
||
|
|
"regularization/advantage_var": 385.0915222167969,
|
||
|
|
"regularization/kl": 0.6411773562431335,
|
||
|
|
"rewards/chosen": 0.5833238318584848,
|
||
|
|
"rewards/margins": 0.8691109942036367,
|
||
|
|
"rewards/rejected": -0.2857871623451519,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 16.640430450439453,
|
||
|
|
"kl": 21.772733688354492,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -35476442.675118856,
|
||
|
|
"logits/rejected": -35618418.39137134,
|
||
|
|
"logps/chosen": -477.05744849445324,
|
||
|
|
"logps/rejected": -355.0386893297381,
|
||
|
|
"loss": 0.4617,
|
||
|
|
"loss/base_kto": 3.1115479469299316,
|
||
|
|
"loss/total_with_kl": 3.6933600902557373,
|
||
|
|
"metrics/advantage_var": 349.43682861328125,
|
||
|
|
"regularization/advantage_var": 349.43682861328125,
|
||
|
|
"regularization/kl": 0.58181232213974,
|
||
|
|
"rewards/chosen": 0.6841963633493463,
|
||
|
|
"rewards/margins": 0.8865140111804446,
|
||
|
|
"rewards/rejected": -0.20231764783109832,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 13.235726356506348,
|
||
|
|
"kl": 15.409103393554688,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -33728573.08634223,
|
||
|
|
"logits/rejected": -34643827.85692068,
|
||
|
|
"logps/chosen": -453.91768053375193,
|
||
|
|
"logps/rejected": -356.26489599533437,
|
||
|
|
"loss": 0.4593,
|
||
|
|
"loss/base_kto": 3.140038251876831,
|
||
|
|
"loss/total_with_kl": 3.6741511821746826,
|
||
|
|
"metrics/advantage_var": 320.7887268066406,
|
||
|
|
"regularization/advantage_var": 320.7887268066406,
|
||
|
|
"regularization/kl": 0.5341132283210754,
|
||
|
|
"rewards/chosen": 0.5848530001318436,
|
||
|
|
"rewards/margins": 0.8610822415341448,
|
||
|
|
"rewards/rejected": -0.2762292414023012,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 20.878890991210938,
|
||
|
|
"kl": 15.422149658203125,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -33578972.57861635,
|
||
|
|
"logits/rejected": -36763638.45962733,
|
||
|
|
"logps/chosen": -471.4190251572327,
|
||
|
|
"logps/rejected": -374.738887810559,
|
||
|
|
"loss": 0.4662,
|
||
|
|
"loss/base_kto": 3.098297119140625,
|
||
|
|
"loss/total_with_kl": 3.729501485824585,
|
||
|
|
"metrics/advantage_var": 379.1016540527344,
|
||
|
|
"regularization/advantage_var": 379.1016540527344,
|
||
|
|
"regularization/kl": 0.6312043070793152,
|
||
|
|
"rewards/chosen": 0.5744507987544222,
|
||
|
|
"rewards/margins": 0.9202158118864583,
|
||
|
|
"rewards/rejected": -0.3457650131320361,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 9.892145156860352,
|
||
|
|
"kl": 13.838292121887207,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -34713857.22098569,
|
||
|
|
"logits/rejected": -36125738.47004608,
|
||
|
|
"logps/chosen": -491.6842209856916,
|
||
|
|
"logps/rejected": -372.671634984639,
|
||
|
|
"loss": 0.4732,
|
||
|
|
"loss/base_kto": 3.0913591384887695,
|
||
|
|
"loss/total_with_kl": 3.7855026721954346,
|
||
|
|
"metrics/advantage_var": 416.90313720703125,
|
||
|
|
"regularization/advantage_var": 416.90313720703125,
|
||
|
|
"regularization/kl": 0.6941437125205994,
|
||
|
|
"rewards/chosen": 0.6123622293881658,
|
||
|
|
"rewards/margins": 0.9411510218708847,
|
||
|
|
"rewards/rejected": -0.3287887924827189,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 16.5397891998291,
|
||
|
|
"kl": 16.939855575561523,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -36120921.56037152,
|
||
|
|
"logits/rejected": -34855603.2807571,
|
||
|
|
"logps/chosen": -469.1806308049536,
|
||
|
|
"logps/rejected": -391.3821963722398,
|
||
|
|
"loss": 0.4673,
|
||
|
|
"loss/base_kto": 3.0890920162200928,
|
||
|
|
"loss/total_with_kl": 3.738292694091797,
|
||
|
|
"metrics/advantage_var": 389.9102478027344,
|
||
|
|
"regularization/advantage_var": 389.9102478027344,
|
||
|
|
"regularization/kl": 0.6492005586624146,
|
||
|
|
"rewards/chosen": 0.5950904113958495,
|
||
|
|
"rewards/margins": 0.9190824456340692,
|
||
|
|
"rewards/rejected": -0.32399203423821965,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 10.007122039794922,
|
||
|
|
"kl": 18.840436935424805,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -34272680.585365854,
|
||
|
|
"logits/rejected": -36623041.64102564,
|
||
|
|
"logps/chosen": -455.6271913109756,
|
||
|
|
"logps/rejected": -372.45587940705127,
|
||
|
|
"loss": 0.4669,
|
||
|
|
"loss/base_kto": 3.1442363262176514,
|
||
|
|
"loss/total_with_kl": 3.7351715564727783,
|
||
|
|
"metrics/advantage_var": 354.916015625,
|
||
|
|
"regularization/advantage_var": 354.916015625,
|
||
|
|
"regularization/kl": 0.590935230255127,
|
||
|
|
"rewards/chosen": 0.6542033683962938,
|
||
|
|
"rewards/margins": 0.8423987681452672,
|
||
|
|
"rewards/rejected": -0.18819539974897337,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 12.622456550598145,
|
||
|
|
"kl": 22.887765884399414,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -36191325.09090909,
|
||
|
|
"logits/rejected": -37970868.02580645,
|
||
|
|
"logps/chosen": -500.6345170454546,
|
||
|
|
"logps/rejected": -361.6321068548387,
|
||
|
|
"loss": 0.4576,
|
||
|
|
"loss/base_kto": 3.096804618835449,
|
||
|
|
"loss/total_with_kl": 3.6607301235198975,
|
||
|
|
"metrics/advantage_var": 338.69384765625,
|
||
|
|
"regularization/advantage_var": 338.69384765625,
|
||
|
|
"regularization/kl": 0.5639252066612244,
|
||
|
|
"rewards/chosen": 0.7195481271454782,
|
||
|
|
"rewards/margins": 0.874699199001577,
|
||
|
|
"rewards/rejected": -0.15515107185609878,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 15.0021333694458,
|
||
|
|
"kl": 23.994291305541992,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -35000614.36085627,
|
||
|
|
"logits/rejected": -36952773.92971246,
|
||
|
|
"logps/chosen": -498.0998184250765,
|
||
|
|
"logps/rejected": -393.1023861821086,
|
||
|
|
"loss": 0.4579,
|
||
|
|
"loss/base_kto": 3.073777675628662,
|
||
|
|
"loss/total_with_kl": 3.663127899169922,
|
||
|
|
"metrics/advantage_var": 353.964111328125,
|
||
|
|
"regularization/advantage_var": 353.964111328125,
|
||
|
|
"regularization/kl": 0.5893502235412598,
|
||
|
|
"rewards/chosen": 0.763191246476013,
|
||
|
|
"rewards/margins": 0.9050856022255638,
|
||
|
|
"rewards/rejected": -0.14189435574955073,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 13.450627326965332,
|
||
|
|
"kl": 18.962284088134766,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -35935625.36585366,
|
||
|
|
"logits/rejected": -37088347.897435896,
|
||
|
|
"logps/chosen": -487.7211318597561,
|
||
|
|
"logps/rejected": -347.68359375,
|
||
|
|
"loss": 0.4654,
|
||
|
|
"loss/base_kto": 3.119527578353882,
|
||
|
|
"loss/total_with_kl": 3.7229549884796143,
|
||
|
|
"metrics/advantage_var": 362.41876220703125,
|
||
|
|
"regularization/advantage_var": 362.41876220703125,
|
||
|
|
"regularization/kl": 0.6034272909164429,
|
||
|
|
"rewards/chosen": 0.6604757076356469,
|
||
|
|
"rewards/margins": 0.8571033245179711,
|
||
|
|
"rewards/rejected": -0.19662761688232422,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 12.219781875610352,
|
||
|
|
"kl": 21.565269470214844,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -34777532.37735849,
|
||
|
|
"logits/rejected": -36161440.596273296,
|
||
|
|
"logps/chosen": -469.9175019654088,
|
||
|
|
"logps/rejected": -363.8256259704969,
|
||
|
|
"loss": 0.4677,
|
||
|
|
"loss/base_kto": 3.0906436443328857,
|
||
|
|
"loss/total_with_kl": 3.7416934967041016,
|
||
|
|
"metrics/advantage_var": 391.020751953125,
|
||
|
|
"regularization/advantage_var": 391.020751953125,
|
||
|
|
"regularization/kl": 0.6510495543479919,
|
||
|
|
"rewards/chosen": 0.7124719199894359,
|
||
|
|
"rewards/margins": 0.9116966408199386,
|
||
|
|
"rewards/rejected": -0.19922472083050272,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 13.32406234741211,
|
||
|
|
"kl": 18.75167465209961,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -35298579.2688172,
|
||
|
|
"logits/rejected": -36003703.24960254,
|
||
|
|
"logps/chosen": -479.2941628264209,
|
||
|
|
"logps/rejected": -384.1168521462639,
|
||
|
|
"loss": 0.467,
|
||
|
|
"loss/base_kto": 3.1590993404388428,
|
||
|
|
"loss/total_with_kl": 3.7356860637664795,
|
||
|
|
"metrics/advantage_var": 346.2985534667969,
|
||
|
|
"regularization/advantage_var": 346.2985534667969,
|
||
|
|
"regularization/kl": 0.5765871405601501,
|
||
|
|
"rewards/chosen": 0.6431738506264401,
|
||
|
|
"rewards/margins": 0.8293609155204931,
|
||
|
|
"rewards/rejected": -0.18618706489405307,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 6.914154529571533,
|
||
|
|
"kl": 18.390832901000977,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -33529994.37837838,
|
||
|
|
"logits/rejected": -36759185.094462544,
|
||
|
|
"logps/chosen": -497.8991647897898,
|
||
|
|
"logps/rejected": -378.38110749185665,
|
||
|
|
"loss": 0.4656,
|
||
|
|
"loss/base_kto": 3.1070802211761475,
|
||
|
|
"loss/total_with_kl": 3.7246978282928467,
|
||
|
|
"metrics/advantage_var": 370.9416198730469,
|
||
|
|
"regularization/advantage_var": 370.9416198730469,
|
||
|
|
"regularization/kl": 0.6176177859306335,
|
||
|
|
"rewards/chosen": 0.6875801430092202,
|
||
|
|
"rewards/margins": 0.9103769279475727,
|
||
|
|
"rewards/rejected": -0.2227967849383525,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 11.438712120056152,
|
||
|
|
"kl": 22.58893394470215,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -34451230.591194965,
|
||
|
|
"logits/rejected": -36731490.583850935,
|
||
|
|
"logps/chosen": -459.84974449685535,
|
||
|
|
"logps/rejected": -372.14358501552795,
|
||
|
|
"loss": 0.4625,
|
||
|
|
"loss/base_kto": 3.102107286453247,
|
||
|
|
"loss/total_with_kl": 3.7000434398651123,
|
||
|
|
"metrics/advantage_var": 359.1208801269531,
|
||
|
|
"regularization/advantage_var": 359.1208801269531,
|
||
|
|
"regularization/kl": 0.5979362726211548,
|
||
|
|
"rewards/chosen": 0.7382436308470912,
|
||
|
|
"rewards/margins": 0.9092386915486391,
|
||
|
|
"rewards/rejected": -0.17099506070154794,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 11.214678764343262,
|
||
|
|
"kl": 18.963571548461914,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -33975406.87898089,
|
||
|
|
"logits/rejected": -34658417.0797546,
|
||
|
|
"logps/chosen": -459.12291003184714,
|
||
|
|
"logps/rejected": -355.2796922929448,
|
||
|
|
"loss": 0.4605,
|
||
|
|
"loss/base_kto": 3.1519150733947754,
|
||
|
|
"loss/total_with_kl": 3.6843254566192627,
|
||
|
|
"metrics/advantage_var": 319.7659912109375,
|
||
|
|
"regularization/advantage_var": 319.7659912109375,
|
||
|
|
"regularization/kl": 0.5324103832244873,
|
||
|
|
"rewards/chosen": 0.6899675016949891,
|
||
|
|
"rewards/margins": 0.853054457227695,
|
||
|
|
"rewards/rejected": -0.16308695553270586,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 9.632356643676758,
|
||
|
|
"kl": 17.70616912841797,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -36399908.45696401,
|
||
|
|
"logits/rejected": -37554153.6349454,
|
||
|
|
"logps/chosen": -495.7028071205008,
|
||
|
|
"logps/rejected": -352.6200760530421,
|
||
|
|
"loss": 0.4641,
|
||
|
|
"loss/base_kto": 3.113957166671753,
|
||
|
|
"loss/total_with_kl": 3.7127532958984375,
|
||
|
|
"metrics/advantage_var": 359.63726806640625,
|
||
|
|
"regularization/advantage_var": 359.63726806640625,
|
||
|
|
"regularization/kl": 0.5987960696220398,
|
||
|
|
"rewards/chosen": 0.6335635953852455,
|
||
|
|
"rewards/margins": 0.8784684452901792,
|
||
|
|
"rewards/rejected": -0.2449048499049337,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 9.035210609436035,
|
||
|
|
"kl": 19.576251983642578,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -36141401.94594595,
|
||
|
|
"logits/rejected": -36757673.674418606,
|
||
|
|
"logps/chosen": -471.6283255912162,
|
||
|
|
"logps/rejected": -411.13349382267444,
|
||
|
|
"loss": 0.4655,
|
||
|
|
"loss/base_kto": 3.1428756713867188,
|
||
|
|
"loss/total_with_kl": 3.724013566970825,
|
||
|
|
"metrics/advantage_var": 349.03204345703125,
|
||
|
|
"regularization/advantage_var": 349.03204345703125,
|
||
|
|
"regularization/kl": 0.5811383128166199,
|
||
|
|
"rewards/chosen": 0.5696450826284047,
|
||
|
|
"rewards/margins": 0.8397197052040315,
|
||
|
|
"rewards/rejected": -0.2700746225756268,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 9.367024421691895,
|
||
|
|
"kl": 15.989706993103027,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -35538447.900621116,
|
||
|
|
"logits/rejected": -36447689.25786164,
|
||
|
|
"logps/chosen": -485.09297360248445,
|
||
|
|
"logps/rejected": -375.897258254717,
|
||
|
|
"loss": 0.4632,
|
||
|
|
"loss/base_kto": 3.1421048641204834,
|
||
|
|
"loss/total_with_kl": 3.7057876586914062,
|
||
|
|
"metrics/advantage_var": 338.54840087890625,
|
||
|
|
"regularization/advantage_var": 338.54840087890625,
|
||
|
|
"regularization/kl": 0.563683032989502,
|
||
|
|
"rewards/chosen": 0.567249179626844,
|
||
|
|
"rewards/margins": 0.8488750592624449,
|
||
|
|
"rewards/rejected": -0.28162587963560093,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 12.35507583618164,
|
||
|
|
"kl": 16.977344512939453,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -35135086.9470305,
|
||
|
|
"logits/rejected": -36150566.57534247,
|
||
|
|
"logps/chosen": -487.79429173354737,
|
||
|
|
"logps/rejected": -385.1545376712329,
|
||
|
|
"loss": 0.4572,
|
||
|
|
"loss/base_kto": 3.122990608215332,
|
||
|
|
"loss/total_with_kl": 3.657369375228882,
|
||
|
|
"metrics/advantage_var": 320.9480895996094,
|
||
|
|
"regularization/advantage_var": 320.9480895996094,
|
||
|
|
"regularization/kl": 0.5343785285949707,
|
||
|
|
"rewards/chosen": 0.5660365121513844,
|
||
|
|
"rewards/margins": 0.8591012508503117,
|
||
|
|
"rewards/rejected": -0.2930647386989274,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 24.943979263305664,
|
||
|
|
"kl": 14.890515327453613,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -36561734.82534776,
|
||
|
|
"logits/rejected": -36859793.99684044,
|
||
|
|
"logps/chosen": -512.2392774343122,
|
||
|
|
"logps/rejected": -367.7308204976303,
|
||
|
|
"loss": 0.4735,
|
||
|
|
"loss/base_kto": 3.0660271644592285,
|
||
|
|
"loss/total_with_kl": 3.7877564430236816,
|
||
|
|
"metrics/advantage_var": 433.4712829589844,
|
||
|
|
"regularization/advantage_var": 433.4712829589844,
|
||
|
|
"regularization/kl": 0.7217296957969666,
|
||
|
|
"rewards/chosen": 0.6244497402373937,
|
||
|
|
"rewards/margins": 0.9588832009249678,
|
||
|
|
"rewards/rejected": -0.33443346068757407,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 12.352337837219238,
|
||
|
|
"kl": 17.1584529876709,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -37212582.4,
|
||
|
|
"logits/rejected": -36345235.2,
|
||
|
|
"logps/chosen": -485.367578125,
|
||
|
|
"logps/rejected": -377.096240234375,
|
||
|
|
"loss": 0.4536,
|
||
|
|
"loss/base_kto": 3.046826124191284,
|
||
|
|
"loss/total_with_kl": 3.628899335861206,
|
||
|
|
"metrics/advantage_var": 349.5934753417969,
|
||
|
|
"regularization/advantage_var": 349.5934753417969,
|
||
|
|
"regularization/kl": 0.5820731520652771,
|
||
|
|
"rewards/chosen": 0.6046004295349121,
|
||
|
|
"rewards/margins": 0.9382753372192383,
|
||
|
|
"rewards/rejected": -0.33367490768432617,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 7.924802303314209,
|
||
|
|
"kl": 16.00665855407715,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -35062070.303030305,
|
||
|
|
"logits/rejected": -36446565.53751914,
|
||
|
|
"logps/chosen": -488.86034688995215,
|
||
|
|
"logps/rejected": -363.7005168453293,
|
||
|
|
"loss": 0.4561,
|
||
|
|
"loss/base_kto": 3.123929738998413,
|
||
|
|
"loss/total_with_kl": 3.648754835128784,
|
||
|
|
"metrics/advantage_var": 315.210205078125,
|
||
|
|
"regularization/advantage_var": 315.210205078125,
|
||
|
|
"regularization/kl": 0.5248250365257263,
|
||
|
|
"rewards/chosen": 0.6202993195212819,
|
||
|
|
"rewards/margins": 0.8590380995210427,
|
||
|
|
"rewards/rejected": -0.23873877999976073,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 13.827885627746582,
|
||
|
|
"kl": 15.445418357849121,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -33636513.514195584,
|
||
|
|
"logits/rejected": -36219774.018575855,
|
||
|
|
"logps/chosen": -457.5471214511041,
|
||
|
|
"logps/rejected": -382.59904702012386,
|
||
|
|
"loss": 0.4622,
|
||
|
|
"loss/base_kto": 3.138892889022827,
|
||
|
|
"loss/total_with_kl": 3.6973092555999756,
|
||
|
|
"metrics/advantage_var": 335.38525390625,
|
||
|
|
"regularization/advantage_var": 335.38525390625,
|
||
|
|
"regularization/kl": 0.558416485786438,
|
||
|
|
"rewards/chosen": 0.5851179538089019,
|
||
|
|
"rewards/margins": 0.8563093884034017,
|
||
|
|
"rewards/rejected": -0.2711914345944998,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 11.497026443481445,
|
||
|
|
"kl": 15.475537300109863,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -36151808.0,
|
||
|
|
"logits/rejected": -36169260.8,
|
||
|
|
"logps/chosen": -468.385791015625,
|
||
|
|
"logps/rejected": -400.343603515625,
|
||
|
|
"loss": 0.4573,
|
||
|
|
"loss/base_kto": 3.059650421142578,
|
||
|
|
"loss/total_with_kl": 3.6584396362304688,
|
||
|
|
"metrics/advantage_var": 359.6334228515625,
|
||
|
|
"regularization/advantage_var": 359.6334228515625,
|
||
|
|
"regularization/kl": 0.598789632320404,
|
||
|
|
"rewards/chosen": 0.6046765804290771,
|
||
|
|
"rewards/margins": 0.9387316703796387,
|
||
|
|
"rewards/rejected": -0.33405508995056155,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 8.553000450134277,
|
||
|
|
"kl": 16.97987937927246,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34956386.461538464,
|
||
|
|
"logits/rejected": -36204500.29268292,
|
||
|
|
"logps/chosen": -490.9650440705128,
|
||
|
|
"logps/rejected": -376.5561880716463,
|
||
|
|
"loss": 0.4666,
|
||
|
|
"loss/base_kto": 3.1323304176330566,
|
||
|
|
"loss/total_with_kl": 3.73240327835083,
|
||
|
|
"metrics/advantage_var": 360.404052734375,
|
||
|
|
"regularization/advantage_var": 360.404052734375,
|
||
|
|
"regularization/kl": 0.6000728011131287,
|
||
|
|
"rewards/chosen": 0.5717535752516526,
|
||
|
|
"rewards/margins": 0.864862864281402,
|
||
|
|
"rewards/rejected": -0.2931092890297494,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 19.277727127075195,
|
||
|
|
"kl": 15.771048545837402,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -34332610.81761006,
|
||
|
|
"logits/rejected": -36033930.335403726,
|
||
|
|
"logps/chosen": -470.49744496855345,
|
||
|
|
"logps/rejected": -371.26353843167703,
|
||
|
|
"loss": 0.4706,
|
||
|
|
"loss/base_kto": 3.1594808101654053,
|
||
|
|
"loss/total_with_kl": 3.7644569873809814,
|
||
|
|
"metrics/advantage_var": 363.3489685058594,
|
||
|
|
"regularization/advantage_var": 363.3489685058594,
|
||
|
|
"regularization/kl": 0.6049759984016418,
|
||
|
|
"rewards/chosen": 0.5511305707055818,
|
||
|
|
"rewards/margins": 0.851343809913353,
|
||
|
|
"rewards/rejected": -0.30021323920777127,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 11.23862361907959,
|
||
|
|
"kl": 15.174311637878418,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -35027817.55450237,
|
||
|
|
"logits/rejected": -35086766.49149923,
|
||
|
|
"logps/chosen": -480.51639020537124,
|
||
|
|
"logps/rejected": -380.3885722565688,
|
||
|
|
"loss": 0.4619,
|
||
|
|
"loss/base_kto": 3.0904994010925293,
|
||
|
|
"loss/total_with_kl": 3.695472002029419,
|
||
|
|
"metrics/advantage_var": 363.34686279296875,
|
||
|
|
"regularization/advantage_var": 363.34686279296875,
|
||
|
|
"regularization/kl": 0.6049725413322449,
|
||
|
|
"rewards/chosen": 0.5956331225933057,
|
||
|
|
"rewards/margins": 0.9028084183921248,
|
||
|
|
"rewards/rejected": -0.3071752957988191,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 7.307109832763672,
|
||
|
|
"kl": 13.383758544921875,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -35780903.68389058,
|
||
|
|
"logits/rejected": -34818821.76205788,
|
||
|
|
"logps/chosen": -477.3777545592705,
|
||
|
|
"logps/rejected": -376.16941318327974,
|
||
|
|
"loss": 0.4591,
|
||
|
|
"loss/base_kto": 3.1255557537078857,
|
||
|
|
"loss/total_with_kl": 3.6724822521209717,
|
||
|
|
"metrics/advantage_var": 328.4842224121094,
|
||
|
|
"regularization/advantage_var": 328.4842224121094,
|
||
|
|
"regularization/kl": 0.5469262003898621,
|
||
|
|
"rewards/chosen": 0.6030227985787899,
|
||
|
|
"rewards/margins": 0.8759559244580102,
|
||
|
|
"rewards/rejected": -0.27293312587922025,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 11.22283935546875,
|
||
|
|
"kl": 15.68100643157959,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -36057814.605965465,
|
||
|
|
"logits/rejected": -37328243.06065319,
|
||
|
|
"logps/chosen": -493.3335949764521,
|
||
|
|
"logps/rejected": -364.83558514774495,
|
||
|
|
"loss": 0.4692,
|
||
|
|
"loss/base_kto": 3.1190693378448486,
|
||
|
|
"loss/total_with_kl": 3.7533156871795654,
|
||
|
|
"metrics/advantage_var": 380.9286804199219,
|
||
|
|
"regularization/advantage_var": 380.9286804199219,
|
||
|
|
"regularization/kl": 0.634246289730072,
|
||
|
|
"rewards/chosen": 0.573127111797243,
|
||
|
|
"rewards/margins": 0.8706419842518016,
|
||
|
|
"rewards/rejected": -0.2975148724545587,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 9.224340438842773,
|
||
|
|
"kl": 15.672713279724121,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -36120357.85055644,
|
||
|
|
"logits/rejected": -36371668.35023042,
|
||
|
|
"logps/chosen": -479.6764705882353,
|
||
|
|
"logps/rejected": -388.6004704301075,
|
||
|
|
"loss": 0.4564,
|
||
|
|
"loss/base_kto": 3.077691078186035,
|
||
|
|
"loss/total_with_kl": 3.650958299636841,
|
||
|
|
"metrics/advantage_var": 344.30462646484375,
|
||
|
|
"regularization/advantage_var": 344.30462646484375,
|
||
|
|
"regularization/kl": 0.5732671618461609,
|
||
|
|
"rewards/chosen": 0.6376804175930296,
|
||
|
|
"rewards/margins": 0.9425419287297423,
|
||
|
|
"rewards/rejected": -0.3048615111367127,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 9.535575866699219,
|
||
|
|
"kl": 20.456087112426758,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -35746735.686274506,
|
||
|
|
"logits/rejected": -36416695.95209581,
|
||
|
|
"logps/chosen": -460.29268790849676,
|
||
|
|
"logps/rejected": -370.57218375748505,
|
||
|
|
"loss": 0.4566,
|
||
|
|
"loss/base_kto": 3.088155508041382,
|
||
|
|
"loss/total_with_kl": 3.652820587158203,
|
||
|
|
"metrics/advantage_var": 339.1382751464844,
|
||
|
|
"regularization/advantage_var": 339.1382751464844,
|
||
|
|
"regularization/kl": 0.5646652579307556,
|
||
|
|
"rewards/chosen": 0.6121223798764297,
|
||
|
|
"rewards/margins": 0.8912944434640981,
|
||
|
|
"rewards/rejected": -0.27917206358766844,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 11.117263793945312,
|
||
|
|
"kl": 13.020883560180664,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -35574272.0,
|
||
|
|
"logits/rejected": -36800747.78947368,
|
||
|
|
"logps/chosen": -501.8949032738095,
|
||
|
|
"logps/rejected": -358.2474814967105,
|
||
|
|
"loss": 0.4607,
|
||
|
|
"loss/base_kto": 3.154110908508301,
|
||
|
|
"loss/total_with_kl": 3.6857082843780518,
|
||
|
|
"metrics/advantage_var": 319.2776794433594,
|
||
|
|
"regularization/advantage_var": 319.2776794433594,
|
||
|
|
"regularization/kl": 0.5315973162651062,
|
||
|
|
"rewards/chosen": 0.5785947527204242,
|
||
|
|
"rewards/margins": 0.8441579897600906,
|
||
|
|
"rewards/rejected": -0.26556323703966644,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.969850461966828e+17,
|
||
|
|
"train_loss": 0.5006774948050938,
|
||
|
|
"train_runtime": 11026.3717,
|
||
|
|
"train_samples_per_second": 13.442,
|
||
|
|
"train_steps_per_second": 0.21
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.969850461966828e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|