2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 17.17824363708496,
|
||
|
|
"kl": 11.705644607543945,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37374555.09202454,
|
||
|
|
"logits/rejected": -38249413.299363054,
|
||
|
|
"logps/chosen": -458.82045628834356,
|
||
|
|
"logps/rejected": -380.9230195063694,
|
||
|
|
"loss": 0.5882,
|
||
|
|
"loss/base_kto": 3.8691139221191406,
|
||
|
|
"metrics/advantage_var": 205.1183624267578,
|
||
|
|
"regularization/mmd": 0.8363999724388123,
|
||
|
|
"regularization/rkhs_norm": 161.1560516357422,
|
||
|
|
"rewards/chosen": 0.19569607600112635,
|
||
|
|
"rewards/margins": 0.12410098730965789,
|
||
|
|
"rewards/rejected": 0.07159508869146845,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 24.634510040283203,
|
||
|
|
"kl": 8.969709396362305,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -36240658.574132495,
|
||
|
|
"logits/rejected": -37311243.888544895,
|
||
|
|
"logps/chosen": -479.26355481072557,
|
||
|
|
"logps/rejected": -362.78703076625385,
|
||
|
|
"loss": 0.5952,
|
||
|
|
"loss/base_kto": 3.939504623413086,
|
||
|
|
"metrics/advantage_var": 254.3942413330078,
|
||
|
|
"regularization/mmd": 0.8221014142036438,
|
||
|
|
"regularization/rkhs_norm": 158.4010467529297,
|
||
|
|
"rewards/chosen": 0.10391850275948221,
|
||
|
|
"rewards/margins": 0.052487514152459426,
|
||
|
|
"rewards/rejected": 0.05143098860702278,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 17.818912506103516,
|
||
|
|
"kl": 9.702555656433105,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -35006144.75294118,
|
||
|
|
"logits/rejected": -36564841.81333333,
|
||
|
|
"logps/chosen": -463.72058823529414,
|
||
|
|
"logps/rejected": -382.8108854166667,
|
||
|
|
"loss": 0.5793,
|
||
|
|
"loss/base_kto": 3.8532512187957764,
|
||
|
|
"metrics/advantage_var": 174.97061157226562,
|
||
|
|
"regularization/mmd": 0.7807949185371399,
|
||
|
|
"regularization/rkhs_norm": 150.4421844482422,
|
||
|
|
"rewards/chosen": 0.1713244494269876,
|
||
|
|
"rewards/margins": 0.13787493630951528,
|
||
|
|
"rewards/rejected": 0.03344951311747233,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 16.61756706237793,
|
||
|
|
"kl": 8.655376434326172,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -35403210.92185008,
|
||
|
|
"logits/rejected": -36740844.00612558,
|
||
|
|
"logps/chosen": -486.4532496012759,
|
||
|
|
"logps/rejected": -385.1654622894334,
|
||
|
|
"loss": 0.5817,
|
||
|
|
"loss/base_kto": 3.8474583625793457,
|
||
|
|
"metrics/advantage_var": 182.4675750732422,
|
||
|
|
"regularization/mmd": 0.8062886595726013,
|
||
|
|
"regularization/rkhs_norm": 155.35426330566406,
|
||
|
|
"rewards/chosen": 0.1467522190708483,
|
||
|
|
"rewards/margins": 0.14803078999961175,
|
||
|
|
"rewards/rejected": -0.001278570928763467,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 17.816631317138672,
|
||
|
|
"kl": 15.2816801071167,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -37329131.11782477,
|
||
|
|
"logits/rejected": -37500411.02912621,
|
||
|
|
"logps/chosen": -480.798291163142,
|
||
|
|
"logps/rejected": -369.70224008899675,
|
||
|
|
"loss": 0.5842,
|
||
|
|
"loss/base_kto": 3.855912923812866,
|
||
|
|
"metrics/advantage_var": 193.5823211669922,
|
||
|
|
"regularization/mmd": 0.8175219893455505,
|
||
|
|
"regularization/rkhs_norm": 157.51869201660156,
|
||
|
|
"rewards/chosen": 0.2912412799016947,
|
||
|
|
"rewards/margins": 0.09936524190898888,
|
||
|
|
"rewards/rejected": 0.1918760379927058,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 12.4163236618042,
|
||
|
|
"kl": 7.135940074920654,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -36614575.15789474,
|
||
|
|
"logits/rejected": -36309686.51104101,
|
||
|
|
"logps/chosen": -471.00386996904024,
|
||
|
|
"logps/rejected": -345.0620317429022,
|
||
|
|
"loss": 0.5811,
|
||
|
|
"loss/base_kto": 3.8355655670166016,
|
||
|
|
"metrics/advantage_var": 193.8709259033203,
|
||
|
|
"regularization/mmd": 0.8132469058036804,
|
||
|
|
"regularization/rkhs_norm": 156.6949920654297,
|
||
|
|
"rewards/chosen": 0.10218977559092614,
|
||
|
|
"rewards/margins": 0.16098254568785844,
|
||
|
|
"rewards/rejected": -0.0587927700969323,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 16.862951278686523,
|
||
|
|
"kl": 5.876062870025635,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -36519189.50237718,
|
||
|
|
"logits/rejected": -37452378.72419106,
|
||
|
|
"logps/chosen": -473.80551703645006,
|
||
|
|
"logps/rejected": -379.79405816640985,
|
||
|
|
"loss": 0.5814,
|
||
|
|
"loss/base_kto": 3.797930955886841,
|
||
|
|
"metrics/advantage_var": 212.62060546875,
|
||
|
|
"regularization/mmd": 0.8534538149833679,
|
||
|
|
"regularization/rkhs_norm": 164.4419708251953,
|
||
|
|
"rewards/chosen": 0.12022552248415214,
|
||
|
|
"rewards/margins": 0.19937941510704252,
|
||
|
|
"rewards/rejected": -0.07915389262289038,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 19.250629425048828,
|
||
|
|
"kl": 8.92680549621582,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36560302.98734177,
|
||
|
|
"logits/rejected": -37849321.87654321,
|
||
|
|
"logps/chosen": -489.98397943037975,
|
||
|
|
"logps/rejected": -368.4852912808642,
|
||
|
|
"loss": 0.5791,
|
||
|
|
"loss/base_kto": 3.8046348094940186,
|
||
|
|
"metrics/advantage_var": 191.76449584960938,
|
||
|
|
"regularization/mmd": 0.8277899026870728,
|
||
|
|
"regularization/rkhs_norm": 159.4970703125,
|
||
|
|
"rewards/chosen": 0.16618209549143345,
|
||
|
|
"rewards/margins": 0.18717863370075993,
|
||
|
|
"rewards/rejected": -0.020996538209326475,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 16.78864288330078,
|
||
|
|
"kl": 5.223937034606934,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -36406336.81012658,
|
||
|
|
"logits/rejected": -38580559.01234568,
|
||
|
|
"logps/chosen": -521.1448773734177,
|
||
|
|
"logps/rejected": -396.00096450617286,
|
||
|
|
"loss": 0.5891,
|
||
|
|
"loss/base_kto": 3.8187339305877686,
|
||
|
|
"metrics/advantage_var": 221.2644805908203,
|
||
|
|
"regularization/mmd": 0.8941717147827148,
|
||
|
|
"regularization/rkhs_norm": 172.2874298095703,
|
||
|
|
"rewards/chosen": 0.01757409904576555,
|
||
|
|
"rewards/margins": 0.17354409555696587,
|
||
|
|
"rewards/rejected": -0.15596999651120033,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 15.423490524291992,
|
||
|
|
"kl": 5.971735000610352,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -36901480.356687896,
|
||
|
|
"logits/rejected": -38046851.926380366,
|
||
|
|
"logps/chosen": -520.1484375,
|
||
|
|
"logps/rejected": -362.76945935582825,
|
||
|
|
"loss": 0.5846,
|
||
|
|
"loss/base_kto": 3.8046326637268066,
|
||
|
|
"metrics/advantage_var": 212.846435546875,
|
||
|
|
"regularization/mmd": 0.8723503947257996,
|
||
|
|
"regularization/rkhs_norm": 168.0829315185547,
|
||
|
|
"rewards/chosen": 0.028980646923089482,
|
||
|
|
"rewards/margins": 0.19316205782374374,
|
||
|
|
"rewards/rejected": -0.16418141090065425,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 17.27155876159668,
|
||
|
|
"kl": 4.341996192932129,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37302842.32911392,
|
||
|
|
"logits/rejected": -37378389.333333336,
|
||
|
|
"logps/chosen": -472.3408821202532,
|
||
|
|
"logps/rejected": -413.896412037037,
|
||
|
|
"loss": 0.5907,
|
||
|
|
"loss/base_kto": 3.857016086578369,
|
||
|
|
"metrics/advantage_var": 219.51280212402344,
|
||
|
|
"regularization/mmd": 0.8689824342727661,
|
||
|
|
"regularization/rkhs_norm": 167.4340057373047,
|
||
|
|
"rewards/chosen": -0.002050889443747605,
|
||
|
|
"rewards/margins": 0.12590938627691936,
|
||
|
|
"rewards/rejected": -0.12796027572066695,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 18.765356063842773,
|
||
|
|
"kl": 13.129290580749512,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -34992805.24006359,
|
||
|
|
"logits/rejected": -34460318.08294931,
|
||
|
|
"logps/chosen": -492.0058624801272,
|
||
|
|
"logps/rejected": -388.18260368663596,
|
||
|
|
"loss": 0.5794,
|
||
|
|
"loss/base_kto": 3.7357583045959473,
|
||
|
|
"metrics/advantage_var": 235.9385223388672,
|
||
|
|
"regularization/mmd": 0.8995633125305176,
|
||
|
|
"regularization/rkhs_norm": 173.32627868652344,
|
||
|
|
"rewards/chosen": 0.2608127321082572,
|
||
|
|
"rewards/margins": 0.2611001880054346,
|
||
|
|
"rewards/rejected": -0.00028745589717741937,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 16.559024810791016,
|
||
|
|
"kl": 11.939021110534668,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -36214281.19760479,
|
||
|
|
"logits/rejected": -37119226.98039216,
|
||
|
|
"logps/chosen": -477.4055950598802,
|
||
|
|
"logps/rejected": -352.95889501633985,
|
||
|
|
"loss": 0.588,
|
||
|
|
"loss/base_kto": 3.7914161682128906,
|
||
|
|
"metrics/advantage_var": 246.98789978027344,
|
||
|
|
"regularization/mmd": 0.9122908711433411,
|
||
|
|
"regularization/rkhs_norm": 175.77857971191406,
|
||
|
|
"rewards/chosen": 0.17018705356620742,
|
||
|
|
"rewards/margins": 0.1747992625372185,
|
||
|
|
"rewards/rejected": -0.004612208971011093,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 18.102062225341797,
|
||
|
|
"kl": 13.486361503601074,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -36812213.52727272,
|
||
|
|
"logits/rejected": -37218687.174193546,
|
||
|
|
"logps/chosen": -485.94038825757576,
|
||
|
|
"logps/rejected": -360.3397933467742,
|
||
|
|
"loss": 0.5759,
|
||
|
|
"loss/base_kto": 3.7146339416503906,
|
||
|
|
"metrics/advantage_var": 225.8873748779297,
|
||
|
|
"regularization/mmd": 0.8924358487129211,
|
||
|
|
"regularization/rkhs_norm": 171.9529571533203,
|
||
|
|
"rewards/chosen": 0.24707082112630208,
|
||
|
|
"rewards/margins": 0.2612232787634737,
|
||
|
|
"rewards/rejected": -0.014152457637171592,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 13.515963554382324,
|
||
|
|
"kl": 7.019232273101807,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -36571723.51377634,
|
||
|
|
"logits/rejected": -36863922.77526395,
|
||
|
|
"logps/chosen": -495.53256685575366,
|
||
|
|
"logps/rejected": -370.2117269984917,
|
||
|
|
"loss": 0.5839,
|
||
|
|
"loss/base_kto": 3.7327675819396973,
|
||
|
|
"metrics/advantage_var": 254.8397216796875,
|
||
|
|
"regularization/mmd": 0.9381042718887329,
|
||
|
|
"regularization/rkhs_norm": 180.75228881835938,
|
||
|
|
"rewards/chosen": 0.08138375305472549,
|
||
|
|
"rewards/margins": 0.24736013788074124,
|
||
|
|
"rewards/rejected": -0.16597638482601573,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 19.815963745117188,
|
||
|
|
"kl": 8.59502124786377,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -36340511.5392,
|
||
|
|
"logits/rejected": -36791547.70076336,
|
||
|
|
"logps/chosen": -467.8384,
|
||
|
|
"logps/rejected": -396.0668893129771,
|
||
|
|
"loss": 0.5894,
|
||
|
|
"loss/base_kto": 3.778308868408203,
|
||
|
|
"metrics/advantage_var": 256.86029052734375,
|
||
|
|
"regularization/mmd": 0.9368079304695129,
|
||
|
|
"regularization/rkhs_norm": 180.5024871826172,
|
||
|
|
"rewards/chosen": 0.115751025390625,
|
||
|
|
"rewards/margins": 0.21329030687171996,
|
||
|
|
"rewards/rejected": -0.09753928148109495,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 16.615589141845703,
|
||
|
|
"kl": 13.181256294250488,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -36934127.53445636,
|
||
|
|
"logits/rejected": -37403290.33492823,
|
||
|
|
"logps/chosen": -466.4159647779479,
|
||
|
|
"logps/rejected": -362.79231459330146,
|
||
|
|
"loss": 0.5853,
|
||
|
|
"loss/base_kto": 3.7623989582061768,
|
||
|
|
"metrics/advantage_var": 241.0950164794922,
|
||
|
|
"regularization/mmd": 0.9203804135322571,
|
||
|
|
"regularization/rkhs_norm": 177.33726501464844,
|
||
|
|
"rewards/chosen": 0.19883580813911753,
|
||
|
|
"rewards/margins": 0.21246085727387848,
|
||
|
|
"rewards/rejected": -0.013625049134760953,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 16.18718147277832,
|
||
|
|
"kl": 9.250204086303711,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -35960675.53140917,
|
||
|
|
"logits/rejected": -36549434.90593343,
|
||
|
|
"logps/chosen": -507.18999363327674,
|
||
|
|
"logps/rejected": -368.66552098408107,
|
||
|
|
"loss": 0.571,
|
||
|
|
"loss/base_kto": 3.7126853466033936,
|
||
|
|
"metrics/advantage_var": 210.2465362548828,
|
||
|
|
"regularization/mmd": 0.855193555355072,
|
||
|
|
"regularization/rkhs_norm": 164.7771759033203,
|
||
|
|
"rewards/chosen": 0.07143040707236842,
|
||
|
|
"rewards/margins": 0.23136470904216988,
|
||
|
|
"rewards/rejected": -0.15993430196980146,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 13.501649856567383,
|
||
|
|
"kl": 7.775170803070068,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -35368973.385620914,
|
||
|
|
"logits/rejected": -35673964.838323355,
|
||
|
|
"logps/chosen": -452.59007352941177,
|
||
|
|
"logps/rejected": -357.5925336826347,
|
||
|
|
"loss": 0.579,
|
||
|
|
"loss/base_kto": 3.742609739303589,
|
||
|
|
"metrics/advantage_var": 225.8137664794922,
|
||
|
|
"regularization/mmd": 0.8894966244697571,
|
||
|
|
"regularization/rkhs_norm": 171.38662719726562,
|
||
|
|
"rewards/chosen": 0.015907189425300148,
|
||
|
|
"rewards/margins": 0.2281374377908737,
|
||
|
|
"rewards/rejected": -0.21223024836557355,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 13.40770149230957,
|
||
|
|
"kl": 1.3833078145980835,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -34698966.28753994,
|
||
|
|
"logits/rejected": -35921036.91743119,
|
||
|
|
"logps/chosen": -480.37569888178916,
|
||
|
|
"logps/rejected": -377.8273126911315,
|
||
|
|
"loss": 0.5851,
|
||
|
|
"loss/base_kto": 3.804372549057007,
|
||
|
|
"metrics/advantage_var": 220.71592712402344,
|
||
|
|
"regularization/mmd": 0.8766388297080994,
|
||
|
|
"regularization/rkhs_norm": 168.90921020507812,
|
||
|
|
"rewards/chosen": -0.1644923298503644,
|
||
|
|
"rewards/margins": 0.18859133609657272,
|
||
|
|
"rewards/rejected": -0.35308366594693713,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 17.20208740234375,
|
||
|
|
"kl": 8.89284896850586,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36301593.17397882,
|
||
|
|
"logits/rejected": -36846570.49434572,
|
||
|
|
"logps/chosen": -484.2128403933434,
|
||
|
|
"logps/rejected": -374.28175484652667,
|
||
|
|
"loss": 0.5773,
|
||
|
|
"loss/base_kto": 3.7007312774658203,
|
||
|
|
"metrics/advantage_var": 252.4306640625,
|
||
|
|
"regularization/mmd": 0.9172918200492859,
|
||
|
|
"regularization/rkhs_norm": 176.74215698242188,
|
||
|
|
"rewards/chosen": 0.2045105488325572,
|
||
|
|
"rewards/margins": 0.2857103797089708,
|
||
|
|
"rewards/rejected": -0.08119983087641357,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 14.008798599243164,
|
||
|
|
"kl": 4.971663951873779,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -35577293.803921565,
|
||
|
|
"logits/rejected": -36158359.76047904,
|
||
|
|
"logps/chosen": -487.22109885620915,
|
||
|
|
"logps/rejected": -380.4648203592814,
|
||
|
|
"loss": 0.5811,
|
||
|
|
"loss/base_kto": 3.7872238159179688,
|
||
|
|
"metrics/advantage_var": 210.58189392089844,
|
||
|
|
"regularization/mmd": 0.8616555333137512,
|
||
|
|
"regularization/rkhs_norm": 166.0222625732422,
|
||
|
|
"rewards/chosen": -0.04419072469075521,
|
||
|
|
"rewards/margins": 0.17861619442998766,
|
||
|
|
"rewards/rejected": -0.22280691912074288,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 29.045827865600586,
|
||
|
|
"kl": 11.158292770385742,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -36004905.99369085,
|
||
|
|
"logits/rejected": -37309325.86996904,
|
||
|
|
"logps/chosen": -495.0600847791798,
|
||
|
|
"logps/rejected": -358.18067917956654,
|
||
|
|
"loss": 0.5785,
|
||
|
|
"loss/base_kto": 3.6740760803222656,
|
||
|
|
"metrics/advantage_var": 253.67251586914062,
|
||
|
|
"regularization/mmd": 0.9537431597709656,
|
||
|
|
"regularization/rkhs_norm": 183.7655487060547,
|
||
|
|
"rewards/chosen": 0.19204202886635574,
|
||
|
|
"rewards/margins": 0.2977901139119488,
|
||
|
|
"rewards/rejected": -0.10574808504559308,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 16.597339630126953,
|
||
|
|
"kl": 6.580680847167969,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -36466796.29237947,
|
||
|
|
"logits/rejected": -36819972.82260597,
|
||
|
|
"logps/chosen": -467.4334175738725,
|
||
|
|
"logps/rejected": -378.80862441130296,
|
||
|
|
"loss": 0.5859,
|
||
|
|
"loss/base_kto": 3.7626149654388428,
|
||
|
|
"metrics/advantage_var": 250.4164276123047,
|
||
|
|
"regularization/mmd": 0.9247032999992371,
|
||
|
|
"regularization/rkhs_norm": 178.1702117919922,
|
||
|
|
"rewards/chosen": 0.05540208238094443,
|
||
|
|
"rewards/margins": 0.21949680070321925,
|
||
|
|
"rewards/rejected": -0.16409471832227482,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 20.037803649902344,
|
||
|
|
"kl": 13.012826919555664,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36411238.23642173,
|
||
|
|
"logits/rejected": -38203529.78593272,
|
||
|
|
"logps/chosen": -506.47623801916933,
|
||
|
|
"logps/rejected": -361.5666093272171,
|
||
|
|
"loss": 0.5786,
|
||
|
|
"loss/base_kto": 3.7578201293945312,
|
||
|
|
"metrics/advantage_var": 226.1883087158203,
|
||
|
|
"regularization/mmd": 0.8712299466133118,
|
||
|
|
"regularization/rkhs_norm": 167.86703491210938,
|
||
|
|
"rewards/chosen": 0.128050173433444,
|
||
|
|
"rewards/margins": 0.19012437972443275,
|
||
|
|
"rewards/rejected": -0.06207420629098875,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 13.235008239746094,
|
||
|
|
"kl": 7.026106357574463,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -35088660.75675676,
|
||
|
|
"logits/rejected": -35795585.76958525,
|
||
|
|
"logps/chosen": -493.02742448330685,
|
||
|
|
"logps/rejected": -365.5026161674347,
|
||
|
|
"loss": 0.5817,
|
||
|
|
"loss/base_kto": 3.6863715648651123,
|
||
|
|
"metrics/advantage_var": 266.0599060058594,
|
||
|
|
"regularization/mmd": 0.9668650031089783,
|
||
|
|
"regularization/rkhs_norm": 186.29383850097656,
|
||
|
|
"rewards/chosen": 0.05598118574327429,
|
||
|
|
"rewards/margins": 0.31304818490622077,
|
||
|
|
"rewards/rejected": -0.25706699916294645,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 14.098347663879395,
|
||
|
|
"kl": 10.816325187683105,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -38242644.79239303,
|
||
|
|
"logits/rejected": -38755597.80585516,
|
||
|
|
"logps/chosen": -504.069235340729,
|
||
|
|
"logps/rejected": -370.5738877118644,
|
||
|
|
"loss": 0.576,
|
||
|
|
"loss/base_kto": 3.670135498046875,
|
||
|
|
"metrics/advantage_var": 260.2480163574219,
|
||
|
|
"regularization/mmd": 0.9379448890686035,
|
||
|
|
"regularization/rkhs_norm": 180.7215576171875,
|
||
|
|
"rewards/chosen": 0.15481238508753467,
|
||
|
|
"rewards/margins": 0.30491048334868165,
|
||
|
|
"rewards/rejected": -0.15009809826114695,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 17.216079711914062,
|
||
|
|
"kl": 10.34858226776123,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -35892446.889589906,
|
||
|
|
"logits/rejected": -37417698.6749226,
|
||
|
|
"logps/chosen": -471.4547515772871,
|
||
|
|
"logps/rejected": -362.39599458204333,
|
||
|
|
"loss": 0.5736,
|
||
|
|
"loss/base_kto": 3.6679515838623047,
|
||
|
|
"metrics/advantage_var": 245.65000915527344,
|
||
|
|
"regularization/mmd": 0.9210807681083679,
|
||
|
|
"regularization/rkhs_norm": 177.4722137451172,
|
||
|
|
"rewards/chosen": 0.16871237529189054,
|
||
|
|
"rewards/margins": 0.3043762965417999,
|
||
|
|
"rewards/rejected": -0.1356639212499093,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 17.89346694946289,
|
||
|
|
"kl": 9.997203826904297,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36724665.26844584,
|
||
|
|
"logits/rejected": -37811551.95023328,
|
||
|
|
"logps/chosen": -470.81063579277867,
|
||
|
|
"logps/rejected": -384.22827566096424,
|
||
|
|
"loss": 0.5802,
|
||
|
|
"loss/base_kto": 3.761526584625244,
|
||
|
|
"metrics/advantage_var": 220.28726196289062,
|
||
|
|
"regularization/mmd": 0.8799467086791992,
|
||
|
|
"regularization/rkhs_norm": 169.54656982421875,
|
||
|
|
"rewards/chosen": 0.19516678694840317,
|
||
|
|
"rewards/margins": 0.22247145854888736,
|
||
|
|
"rewards/rejected": -0.027304671600484182,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 12.943658828735352,
|
||
|
|
"kl": 9.310834884643555,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -36783370.432601884,
|
||
|
|
"logits/rejected": -38423248.9470405,
|
||
|
|
"logps/chosen": -502.1896551724138,
|
||
|
|
"logps/rejected": -381.56746495327104,
|
||
|
|
"loss": 0.5845,
|
||
|
|
"loss/base_kto": 3.736865997314453,
|
||
|
|
"metrics/advantage_var": 248.659912109375,
|
||
|
|
"regularization/mmd": 0.939008891582489,
|
||
|
|
"regularization/rkhs_norm": 180.9265594482422,
|
||
|
|
"rewards/chosen": 0.13101956836855896,
|
||
|
|
"rewards/margins": 0.24130708911290646,
|
||
|
|
"rewards/rejected": -0.1102875207443475,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 25.93958854675293,
|
||
|
|
"kl": 9.676671028137207,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36023722.666666664,
|
||
|
|
"logits/rejected": -37273651.848101266,
|
||
|
|
"logps/chosen": -511.7439236111111,
|
||
|
|
"logps/rejected": -354.8786095727848,
|
||
|
|
"loss": 0.5797,
|
||
|
|
"loss/base_kto": 3.6770730018615723,
|
||
|
|
"metrics/advantage_var": 268.8786315917969,
|
||
|
|
"regularization/mmd": 0.9602136015892029,
|
||
|
|
"regularization/rkhs_norm": 185.0122528076172,
|
||
|
|
"rewards/chosen": 0.15822723765432098,
|
||
|
|
"rewards/margins": 0.28441497262035764,
|
||
|
|
"rewards/rejected": -0.12618773496603664,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 18.380504608154297,
|
||
|
|
"kl": 17.58591651916504,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -35677755.96396396,
|
||
|
|
"logits/rejected": -36941306.99674267,
|
||
|
|
"logps/chosen": -476.466966966967,
|
||
|
|
"logps/rejected": -369.57168668566777,
|
||
|
|
"loss": 0.5804,
|
||
|
|
"loss/base_kto": 3.703371524810791,
|
||
|
|
"metrics/advantage_var": 253.0450439453125,
|
||
|
|
"regularization/mmd": 0.940221905708313,
|
||
|
|
"regularization/rkhs_norm": 181.16029357910156,
|
||
|
|
"rewards/chosen": 0.3106857620560013,
|
||
|
|
"rewards/margins": 0.24153146378932022,
|
||
|
|
"rewards/rejected": 0.06915429826668108,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 12.880158424377441,
|
||
|
|
"kl": 11.941235542297363,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36219840.18991098,
|
||
|
|
"logits/rejected": -36503883.19471947,
|
||
|
|
"logps/chosen": -472.118462537092,
|
||
|
|
"logps/rejected": -382.3772174092409,
|
||
|
|
"loss": 0.5795,
|
||
|
|
"loss/base_kto": 3.7101731300354004,
|
||
|
|
"metrics/advantage_var": 242.2503662109375,
|
||
|
|
"regularization/mmd": 0.9256561398506165,
|
||
|
|
"regularization/rkhs_norm": 178.35379028320312,
|
||
|
|
"rewards/chosen": 0.24291000592602466,
|
||
|
|
"rewards/margins": 0.2575938961900511,
|
||
|
|
"rewards/rejected": -0.014683890264026403,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 17.78050422668457,
|
||
|
|
"kl": 17.598390579223633,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -36216828.94328358,
|
||
|
|
"logits/rejected": -36368175.84262295,
|
||
|
|
"logps/chosen": -487.8546641791045,
|
||
|
|
"logps/rejected": -374.89918032786886,
|
||
|
|
"loss": 0.5805,
|
||
|
|
"loss/base_kto": 3.708536148071289,
|
||
|
|
"metrics/advantage_var": 246.02774047851562,
|
||
|
|
"regularization/mmd": 0.9358450174331665,
|
||
|
|
"regularization/rkhs_norm": 180.3169708251953,
|
||
|
|
"rewards/chosen": 0.33828120445137594,
|
||
|
|
"rewards/margins": 0.23322944463772327,
|
||
|
|
"rewards/rejected": 0.10505175981365267,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 17.22652816772461,
|
||
|
|
"kl": 15.75074291229248,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -36608895.59874608,
|
||
|
|
"logits/rejected": -37650680.8224299,
|
||
|
|
"logps/chosen": -504.24794278996865,
|
||
|
|
"logps/rejected": -397.5719431464174,
|
||
|
|
"loss": 0.579,
|
||
|
|
"loss/base_kto": 3.734424591064453,
|
||
|
|
"metrics/advantage_var": 258.8893737792969,
|
||
|
|
"regularization/mmd": 0.8979118466377258,
|
||
|
|
"regularization/rkhs_norm": 173.00808715820312,
|
||
|
|
"rewards/chosen": 0.2579734586996718,
|
||
|
|
"rewards/margins": 0.22677540275496166,
|
||
|
|
"rewards/rejected": 0.031198055944710135,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 19.103612899780273,
|
||
|
|
"kl": 16.009614944458008,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -35493088.39506173,
|
||
|
|
"logits/rejected": -37134031.39240506,
|
||
|
|
"logps/chosen": -489.5078125,
|
||
|
|
"logps/rejected": -360.2166485363924,
|
||
|
|
"loss": 0.5729,
|
||
|
|
"loss/base_kto": 3.6622560024261475,
|
||
|
|
"metrics/advantage_var": 238.2059326171875,
|
||
|
|
"regularization/mmd": 0.9208865165710449,
|
||
|
|
"regularization/rkhs_norm": 177.4347686767578,
|
||
|
|
"rewards/chosen": 0.2761618767255618,
|
||
|
|
"rewards/margins": 0.2882395832496502,
|
||
|
|
"rewards/rejected": -0.012077706524088412,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 14.776968002319336,
|
||
|
|
"kl": 9.46393871307373,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36609194.666666664,
|
||
|
|
"logits/rejected": -36110916.682926826,
|
||
|
|
"logps/chosen": -485.6818409455128,
|
||
|
|
"logps/rejected": -383.7903248856707,
|
||
|
|
"loss": 0.5848,
|
||
|
|
"loss/base_kto": 3.771756410598755,
|
||
|
|
"metrics/advantage_var": 239.65869140625,
|
||
|
|
"regularization/mmd": 0.906679630279541,
|
||
|
|
"regularization/rkhs_norm": 174.6974334716797,
|
||
|
|
"rewards/chosen": 0.08732181940323268,
|
||
|
|
"rewards/margins": 0.18117325778600946,
|
||
|
|
"rewards/rejected": -0.09385143838277678,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 22.42934799194336,
|
||
|
|
"kl": 20.882888793945312,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -35708801.69162995,
|
||
|
|
"logits/rejected": -36525840.66777963,
|
||
|
|
"logps/chosen": -487.431718061674,
|
||
|
|
"logps/rejected": -368.09416736227047,
|
||
|
|
"loss": 0.5788,
|
||
|
|
"loss/base_kto": 3.6808204650878906,
|
||
|
|
"metrics/advantage_var": 252.8956298828125,
|
||
|
|
"regularization/mmd": 0.9492565989494324,
|
||
|
|
"regularization/rkhs_norm": 182.90109252929688,
|
||
|
|
"rewards/chosen": 0.37040340742875827,
|
||
|
|
"rewards/margins": 0.30396737859000356,
|
||
|
|
"rewards/rejected": 0.0664360288387547,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 18.60274887084961,
|
||
|
|
"kl": 18.21516227722168,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -36211937.57210776,
|
||
|
|
"logits/rejected": -36369403.251931995,
|
||
|
|
"logps/chosen": -469.4092709984152,
|
||
|
|
"logps/rejected": -361.03458268933537,
|
||
|
|
"loss": 0.5747,
|
||
|
|
"loss/base_kto": 3.5325393676757812,
|
||
|
|
"metrics/advantage_var": 381.392822265625,
|
||
|
|
"regularization/mmd": 1.0648847818374634,
|
||
|
|
"regularization/rkhs_norm": 205.18008422851562,
|
||
|
|
"rewards/chosen": 0.42569472710417494,
|
||
|
|
"rewards/margins": 0.4627660081953137,
|
||
|
|
"rewards/rejected": -0.03707128109113879,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 14.687518119812012,
|
||
|
|
"kl": 14.834124565124512,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -36750019.08553655,
|
||
|
|
"logits/rejected": -38123317.45054945,
|
||
|
|
"logps/chosen": -482.89434292379474,
|
||
|
|
"logps/rejected": -380.6186715070644,
|
||
|
|
"loss": 0.5745,
|
||
|
|
"loss/base_kto": 3.207934617996216,
|
||
|
|
"metrics/advantage_var": 620.0404663085938,
|
||
|
|
"regularization/mmd": 1.3884562253952026,
|
||
|
|
"regularization/rkhs_norm": 267.5252990722656,
|
||
|
|
"rewards/chosen": 0.4830367828416602,
|
||
|
|
"rewards/margins": 0.8749562886811053,
|
||
|
|
"rewards/rejected": -0.3919195058394451,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 16.64634132385254,
|
||
|
|
"kl": 19.01596450805664,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -36565908.54320987,
|
||
|
|
"logits/rejected": -38423234.43037975,
|
||
|
|
"logps/chosen": -491.9679783950617,
|
||
|
|
"logps/rejected": -356.3671380537975,
|
||
|
|
"loss": 0.5666,
|
||
|
|
"loss/base_kto": 3.2073490619659424,
|
||
|
|
"metrics/advantage_var": 536.5540161132812,
|
||
|
|
"regularization/mmd": 1.325656533241272,
|
||
|
|
"regularization/rkhs_norm": 255.42515563964844,
|
||
|
|
"rewards/chosen": 0.6128322930983555,
|
||
|
|
"rewards/margins": 0.8244463948462788,
|
||
|
|
"rewards/rejected": -0.21161410174792325,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 15.253790855407715,
|
||
|
|
"kl": 14.648599624633789,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -36932947.2,
|
||
|
|
"logits/rejected": -37453164.8,
|
||
|
|
"logps/chosen": -471.4841796875,
|
||
|
|
"logps/rejected": -396.698095703125,
|
||
|
|
"loss": 0.5787,
|
||
|
|
"loss/base_kto": 3.218859910964966,
|
||
|
|
"metrics/advantage_var": 649.7261962890625,
|
||
|
|
"regularization/mmd": 1.4109556674957275,
|
||
|
|
"regularization/rkhs_norm": 271.8604431152344,
|
||
|
|
"rewards/chosen": 0.5692643165588379,
|
||
|
|
"rewards/margins": 0.8850962162017822,
|
||
|
|
"rewards/rejected": -0.31583189964294434,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 15.689753532409668,
|
||
|
|
"kl": 16.705398559570312,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -35548578.6163522,
|
||
|
|
"logits/rejected": -37207968.596273296,
|
||
|
|
"logps/chosen": -497.97297562893084,
|
||
|
|
"logps/rejected": -377.9689926242236,
|
||
|
|
"loss": 0.5644,
|
||
|
|
"loss/base_kto": 3.212815046310425,
|
||
|
|
"metrics/advantage_var": 559.6679077148438,
|
||
|
|
"regularization/mmd": 1.3021559715270996,
|
||
|
|
"regularization/rkhs_norm": 250.8970947265625,
|
||
|
|
"rewards/chosen": 0.5622371457657724,
|
||
|
|
"rewards/margins": 0.861396983571479,
|
||
|
|
"rewards/rejected": -0.29915983780570654,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 12.39108943939209,
|
||
|
|
"kl": 12.886560440063477,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -35173405.79479326,
|
||
|
|
"logits/rejected": -36652200.2169059,
|
||
|
|
"logps/chosen": -445.79383614088823,
|
||
|
|
"logps/rejected": -382.7857605661882,
|
||
|
|
"loss": 0.5668,
|
||
|
|
"loss/base_kto": 3.203948974609375,
|
||
|
|
"metrics/advantage_var": 564.2401733398438,
|
||
|
|
"regularization/mmd": 1.3306390047073364,
|
||
|
|
"regularization/rkhs_norm": 256.38519287109375,
|
||
|
|
"rewards/chosen": 0.49435803353512636,
|
||
|
|
"rewards/margins": 0.8814675496850093,
|
||
|
|
"rewards/rejected": -0.38710951614988287,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 17.558612823486328,
|
||
|
|
"kl": 14.927868843078613,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -35741001.544740975,
|
||
|
|
"logits/rejected": -36535453.66096423,
|
||
|
|
"logps/chosen": -470.7338598901099,
|
||
|
|
"logps/rejected": -370.9288005443235,
|
||
|
|
"loss": 0.5646,
|
||
|
|
"loss/base_kto": 3.196678876876831,
|
||
|
|
"metrics/advantage_var": 581.9120483398438,
|
||
|
|
"regularization/mmd": 1.3197563886642456,
|
||
|
|
"regularization/rkhs_norm": 254.288330078125,
|
||
|
|
"rewards/chosen": 0.5868753526049842,
|
||
|
|
"rewards/margins": 0.8817949020041629,
|
||
|
|
"rewards/rejected": -0.29491954939917864,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 12.255751609802246,
|
||
|
|
"kl": 10.112584114074707,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -33169847.511177346,
|
||
|
|
"logits/rejected": -35017289.14285714,
|
||
|
|
"logps/chosen": -476.17688152011925,
|
||
|
|
"logps/rejected": -369.58138341543514,
|
||
|
|
"loss": 0.5696,
|
||
|
|
"loss/base_kto": 3.2321934700012207,
|
||
|
|
"metrics/advantage_var": 565.8189697265625,
|
||
|
|
"regularization/mmd": 1.3248211145401,
|
||
|
|
"regularization/rkhs_norm": 255.2642059326172,
|
||
|
|
"rewards/chosen": 0.5160929058774217,
|
||
|
|
"rewards/margins": 0.850682170344658,
|
||
|
|
"rewards/rejected": -0.33458926446723625,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 12.581541061401367,
|
||
|
|
"kl": 21.432767868041992,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35019660.53412463,
|
||
|
|
"logits/rejected": -35742821.38613861,
|
||
|
|
"logps/chosen": -492.2127225519288,
|
||
|
|
"logps/rejected": -366.06311881188117,
|
||
|
|
"loss": 0.5741,
|
||
|
|
"loss/base_kto": 3.2351837158203125,
|
||
|
|
"metrics/advantage_var": 610.61083984375,
|
||
|
|
"regularization/mmd": 1.3577238321304321,
|
||
|
|
"regularization/rkhs_norm": 261.6038513183594,
|
||
|
|
"rewards/chosen": 0.6210258325412649,
|
||
|
|
"rewards/margins": 0.8083742518314361,
|
||
|
|
"rewards/rejected": -0.1873484192901712,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 11.823034286499023,
|
||
|
|
"kl": 16.603158950805664,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -34504306.14012739,
|
||
|
|
"logits/rejected": -34836454.87116564,
|
||
|
|
"logps/chosen": -494.34280453821657,
|
||
|
|
"logps/rejected": -389.03575536809814,
|
||
|
|
"loss": 0.5565,
|
||
|
|
"loss/base_kto": 3.244631290435791,
|
||
|
|
"metrics/advantage_var": 445.3918151855469,
|
||
|
|
"regularization/mmd": 1.2072175741195679,
|
||
|
|
"regularization/rkhs_norm": 232.6045379638672,
|
||
|
|
"rewards/chosen": 0.5095137577907295,
|
||
|
|
"rewards/margins": 0.8027251038143827,
|
||
|
|
"rewards/rejected": -0.2932113460236532,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 12.478619575500488,
|
||
|
|
"kl": 16.44584083557129,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -34616902.51497006,
|
||
|
|
"logits/rejected": -37056214.16993464,
|
||
|
|
"logps/chosen": -486.3734094311377,
|
||
|
|
"logps/rejected": -356.7321793300654,
|
||
|
|
"loss": 0.5713,
|
||
|
|
"loss/base_kto": 3.161914348602295,
|
||
|
|
"metrics/advantage_var": 634.0953979492188,
|
||
|
|
"regularization/mmd": 1.4081577062606812,
|
||
|
|
"regularization/rkhs_norm": 271.32135009765625,
|
||
|
|
"rewards/chosen": 0.6529807816008608,
|
||
|
|
"rewards/margins": 0.9607664640297569,
|
||
|
|
"rewards/rejected": -0.30778568242889603,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 12.8961763381958,
|
||
|
|
"kl": 22.179479598999023,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34196191.65905631,
|
||
|
|
"logits/rejected": -35577073.61797753,
|
||
|
|
"logps/chosen": -489.5189307458143,
|
||
|
|
"logps/rejected": -364.6163723916533,
|
||
|
|
"loss": 0.5606,
|
||
|
|
"loss/base_kto": 3.1646974086761475,
|
||
|
|
"metrics/advantage_var": 525.2611694335938,
|
||
|
|
"regularization/mmd": 1.31988525390625,
|
||
|
|
"regularization/rkhs_norm": 254.31314086914062,
|
||
|
|
"rewards/chosen": 0.7416043680733924,
|
||
|
|
"rewards/margins": 0.875811663449847,
|
||
|
|
"rewards/rejected": -0.13420729537645465,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 16.533170700073242,
|
||
|
|
"kl": 14.972345352172852,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -34331405.2621664,
|
||
|
|
"logits/rejected": -34009075.259720065,
|
||
|
|
"logps/chosen": -473.5820251177394,
|
||
|
|
"logps/rejected": -356.50770314930014,
|
||
|
|
"loss": 0.5626,
|
||
|
|
"loss/base_kto": 3.216235876083374,
|
||
|
|
"metrics/advantage_var": 530.1517333984375,
|
||
|
|
"regularization/mmd": 1.2849127054214478,
|
||
|
|
"regularization/rkhs_norm": 247.57470703125,
|
||
|
|
"rewards/chosen": 0.5380662951027767,
|
||
|
|
"rewards/margins": 0.8503745991493601,
|
||
|
|
"rewards/rejected": -0.3123083040465834,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 8.892478942871094,
|
||
|
|
"kl": 11.803365707397461,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -34360079.441269845,
|
||
|
|
"logits/rejected": -34933177.10769231,
|
||
|
|
"logps/chosen": -480.28035714285716,
|
||
|
|
"logps/rejected": -374.7328365384615,
|
||
|
|
"loss": 0.5574,
|
||
|
|
"loss/base_kto": 3.1655123233795166,
|
||
|
|
"metrics/advantage_var": 536.4296875,
|
||
|
|
"regularization/mmd": 1.2939534187316895,
|
||
|
|
"regularization/rkhs_norm": 249.316650390625,
|
||
|
|
"rewards/chosen": 0.5368955581907242,
|
||
|
|
"rewards/margins": 0.9317764926859164,
|
||
|
|
"rewards/rejected": -0.3948809344951923,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 11.752116203308105,
|
||
|
|
"kl": 9.889144897460938,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -36479127.7037037,
|
||
|
|
"logits/rejected": -36003818.24582701,
|
||
|
|
"logps/chosen": -520.2193035426731,
|
||
|
|
"logps/rejected": -378.63972401365703,
|
||
|
|
"loss": 0.5772,
|
||
|
|
"loss/base_kto": 3.2081775665283203,
|
||
|
|
"metrics/advantage_var": 663.02197265625,
|
||
|
|
"regularization/mmd": 1.4091278314590454,
|
||
|
|
"regularization/rkhs_norm": 271.5082702636719,
|
||
|
|
"rewards/chosen": 0.4727240668402778,
|
||
|
|
"rewards/margins": 0.8728357269076147,
|
||
|
|
"rewards/rejected": -0.4001116600673369,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 16.082019805908203,
|
||
|
|
"kl": 6.320364475250244,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34601571.39314845,
|
||
|
|
"logits/rejected": -35609131.754122935,
|
||
|
|
"logps/chosen": -462.0130505709625,
|
||
|
|
"logps/rejected": -380.0276424287856,
|
||
|
|
"loss": 0.5659,
|
||
|
|
"loss/base_kto": 3.2715508937835693,
|
||
|
|
"metrics/advantage_var": 455.0561218261719,
|
||
|
|
"regularization/mmd": 1.255906105041504,
|
||
|
|
"regularization/rkhs_norm": 241.98574829101562,
|
||
|
|
"rewards/chosen": 0.30424208275451164,
|
||
|
|
"rewards/margins": 0.7885519297841219,
|
||
|
|
"rewards/rejected": -0.4843098470296102,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 17.883174896240234,
|
||
|
|
"kl": 8.331497192382812,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34640315.94936709,
|
||
|
|
"logits/rejected": -36164487.90123457,
|
||
|
|
"logps/chosen": -491.1646558544304,
|
||
|
|
"logps/rejected": -387.675106095679,
|
||
|
|
"loss": 0.5795,
|
||
|
|
"loss/base_kto": 3.169969320297241,
|
||
|
|
"metrics/advantage_var": 712.0158081054688,
|
||
|
|
"regularization/mmd": 1.466303825378418,
|
||
|
|
"regularization/rkhs_norm": 282.5248107910156,
|
||
|
|
"rewards/chosen": 0.41718383982211726,
|
||
|
|
"rewards/margins": 0.9380789618321481,
|
||
|
|
"rewards/rejected": -0.5208951220100309,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 13.120745658874512,
|
||
|
|
"kl": 11.672740936279297,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -34248807.206299216,
|
||
|
|
"logits/rejected": -35292952.21085271,
|
||
|
|
"logps/chosen": -453.683907480315,
|
||
|
|
"logps/rejected": -369.9968265503876,
|
||
|
|
"loss": 0.5667,
|
||
|
|
"loss/base_kto": 3.268097400665283,
|
||
|
|
"metrics/advantage_var": 539.9569091796875,
|
||
|
|
"regularization/mmd": 1.2653586864471436,
|
||
|
|
"regularization/rkhs_norm": 243.8070831298828,
|
||
|
|
"rewards/chosen": 0.37055841881459156,
|
||
|
|
"rewards/margins": 0.7732348340731914,
|
||
|
|
"rewards/rejected": -0.4026764152585998,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 14.594772338867188,
|
||
|
|
"kl": 12.012019157409668,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -34129042.73651772,
|
||
|
|
"logits/rejected": -34942648.19017433,
|
||
|
|
"logps/chosen": -471.9741910631741,
|
||
|
|
"logps/rejected": -360.9488411251981,
|
||
|
|
"loss": 0.5646,
|
||
|
|
"loss/base_kto": 3.235393524169922,
|
||
|
|
"metrics/advantage_var": 526.1349487304688,
|
||
|
|
"regularization/mmd": 1.2810367345809937,
|
||
|
|
"regularization/rkhs_norm": 246.82791137695312,
|
||
|
|
"rewards/chosen": 0.5198080242139349,
|
||
|
|
"rewards/margins": 0.8130445405709699,
|
||
|
|
"rewards/rejected": -0.29323651635703496,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 10.872600555419922,
|
||
|
|
"kl": 16.5655460357666,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -35674686.23662885,
|
||
|
|
"logits/rejected": -36162842.642533936,
|
||
|
|
"logps/chosen": -480.79092382495946,
|
||
|
|
"logps/rejected": -369.57614536199094,
|
||
|
|
"loss": 0.5589,
|
||
|
|
"loss/base_kto": 3.2694852352142334,
|
||
|
|
"metrics/advantage_var": 443.4609069824219,
|
||
|
|
"regularization/mmd": 1.2020606994628906,
|
||
|
|
"regularization/rkhs_norm": 231.61094665527344,
|
||
|
|
"rewards/chosen": 0.4575632549761953,
|
||
|
|
"rewards/margins": 0.7811450023160568,
|
||
|
|
"rewards/rejected": -0.32358174733986145,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 25.44319725036621,
|
||
|
|
"kl": 11.072982788085938,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34653834.48531685,
|
||
|
|
"logits/rejected": -35809241.17535545,
|
||
|
|
"logps/chosen": -484.25661707882534,
|
||
|
|
"logps/rejected": -367.08656694312793,
|
||
|
|
"loss": 0.5779,
|
||
|
|
"loss/base_kto": 3.2494781017303467,
|
||
|
|
"metrics/advantage_var": 722.969970703125,
|
||
|
|
"regularization/mmd": 1.3739899396896362,
|
||
|
|
"regularization/rkhs_norm": 264.7379455566406,
|
||
|
|
"rewards/chosen": 0.4823256054838437,
|
||
|
|
"rewards/margins": 0.8268611318016631,
|
||
|
|
"rewards/rejected": -0.3445355263178194,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 11.706360816955566,
|
||
|
|
"kl": 6.393588542938232,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -34604419.97515528,
|
||
|
|
"logits/rejected": -36769808.10062893,
|
||
|
|
"logps/chosen": -475.4421583850932,
|
||
|
|
"logps/rejected": -392.9174282625786,
|
||
|
|
"loss": 0.5581,
|
||
|
|
"loss/base_kto": 3.1633262634277344,
|
||
|
|
"metrics/advantage_var": 502.78985595703125,
|
||
|
|
"regularization/mmd": 1.3012621402740479,
|
||
|
|
"regularization/rkhs_norm": 250.7248992919922,
|
||
|
|
"rewards/chosen": 0.39339698175465837,
|
||
|
|
"rewards/margins": 0.9010490245514351,
|
||
|
|
"rewards/rejected": -0.5076520427967768,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 15.006986618041992,
|
||
|
|
"kl": 13.777557373046875,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -35110965.36807817,
|
||
|
|
"logits/rejected": -35664573.117117114,
|
||
|
|
"logps/chosen": -485.68800895765474,
|
||
|
|
"logps/rejected": -372.91446133633633,
|
||
|
|
"loss": 0.5656,
|
||
|
|
"loss/base_kto": 3.1921632289886475,
|
||
|
|
"metrics/advantage_var": 620.4743041992188,
|
||
|
|
"regularization/mmd": 1.3327293395996094,
|
||
|
|
"regularization/rkhs_norm": 256.7879333496094,
|
||
|
|
"rewards/chosen": 0.5097670166810871,
|
||
|
|
"rewards/margins": 0.9058117735962055,
|
||
|
|
"rewards/rejected": -0.39604475691511826,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 13.3948335647583,
|
||
|
|
"kl": 12.922057151794434,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -34602487.37519143,
|
||
|
|
"logits/rejected": -36220268.197767146,
|
||
|
|
"logps/chosen": -473.24832503828486,
|
||
|
|
"logps/rejected": -379.8467653508772,
|
||
|
|
"loss": 0.5714,
|
||
|
|
"loss/base_kto": 3.195450782775879,
|
||
|
|
"metrics/advantage_var": 598.126953125,
|
||
|
|
"regularization/mmd": 1.3760026693344116,
|
||
|
|
"regularization/rkhs_norm": 265.1257629394531,
|
||
|
|
"rewards/chosen": 0.4908813196155963,
|
||
|
|
"rewards/margins": 0.8503444166114595,
|
||
|
|
"rewards/rejected": -0.35946309699586326,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 10.40219783782959,
|
||
|
|
"kl": 13.461359024047852,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -35307949.627760254,
|
||
|
|
"logits/rejected": -37240178.92260062,
|
||
|
|
"logps/chosen": -503.9521884858044,
|
||
|
|
"logps/rejected": -370.43815305727554,
|
||
|
|
"loss": 0.5666,
|
||
|
|
"loss/base_kto": 3.1887176036834717,
|
||
|
|
"metrics/advantage_var": 568.2356567382812,
|
||
|
|
"regularization/mmd": 1.3442790508270264,
|
||
|
|
"regularization/rkhs_norm": 259.0133361816406,
|
||
|
|
"rewards/chosen": 0.554238496894716,
|
||
|
|
"rewards/margins": 0.8942932716957754,
|
||
|
|
"rewards/rejected": -0.3400547748010594,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 13.56784439086914,
|
||
|
|
"kl": 12.849693298339844,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -35670589.44,
|
||
|
|
"logits/rejected": -35388243.70793651,
|
||
|
|
"logps/chosen": -471.4319711538462,
|
||
|
|
"logps/rejected": -390.88993055555557,
|
||
|
|
"loss": 0.5605,
|
||
|
|
"loss/base_kto": 3.197096347808838,
|
||
|
|
"metrics/advantage_var": 507.1678771972656,
|
||
|
|
"regularization/mmd": 1.2867193222045898,
|
||
|
|
"regularization/rkhs_norm": 247.9228057861328,
|
||
|
|
"rewards/chosen": 0.46277982271634616,
|
||
|
|
"rewards/margins": 0.8514659392528046,
|
||
|
|
"rewards/rejected": -0.38868611653645835,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 11.178312301635742,
|
||
|
|
"kl": 13.838495254516602,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -35516440.23668639,
|
||
|
|
"logits/rejected": -35471732.98013245,
|
||
|
|
"logps/chosen": -480.5735022189349,
|
||
|
|
"logps/rejected": -377.0640004139073,
|
||
|
|
"loss": 0.5584,
|
||
|
|
"loss/base_kto": 3.1743791103363037,
|
||
|
|
"metrics/advantage_var": 519.8939208984375,
|
||
|
|
"regularization/mmd": 1.2927216291427612,
|
||
|
|
"regularization/rkhs_norm": 249.0792999267578,
|
||
|
|
"rewards/chosen": 0.6114454100117881,
|
||
|
|
"rewards/margins": 0.8495071001844003,
|
||
|
|
"rewards/rejected": -0.23806169017261228,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 13.94364070892334,
|
||
|
|
"kl": 13.35486888885498,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -35970627.654159866,
|
||
|
|
"logits/rejected": -35843678.4167916,
|
||
|
|
"logps/chosen": -480.89228181076675,
|
||
|
|
"logps/rejected": -384.869049850075,
|
||
|
|
"loss": 0.5568,
|
||
|
|
"loss/base_kto": 3.111708879470825,
|
||
|
|
"metrics/advantage_var": 580.8645629882812,
|
||
|
|
"regularization/mmd": 1.3426902294158936,
|
||
|
|
"regularization/rkhs_norm": 258.7071838378906,
|
||
|
|
"rewards/chosen": 0.5695681797544097,
|
||
|
|
"rewards/margins": 0.9661051712684183,
|
||
|
|
"rewards/rejected": -0.3965369915140086,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 14.192017555236816,
|
||
|
|
"kl": 9.430925369262695,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -34147431.69620253,
|
||
|
|
"logits/rejected": -35879531.45679013,
|
||
|
|
"logps/chosen": -483.69882318037975,
|
||
|
|
"logps/rejected": -362.12548225308643,
|
||
|
|
"loss": 0.5626,
|
||
|
|
"loss/base_kto": 3.2272446155548096,
|
||
|
|
"metrics/advantage_var": 537.1535034179688,
|
||
|
|
"regularization/mmd": 1.2738579511642456,
|
||
|
|
"regularization/rkhs_norm": 245.44468688964844,
|
||
|
|
"rewards/chosen": 0.40774927260000493,
|
||
|
|
"rewards/margins": 0.8499849131375965,
|
||
|
|
"rewards/rejected": -0.4422356405375916,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 13.242347717285156,
|
||
|
|
"kl": 12.410801887512207,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -35401925.16770186,
|
||
|
|
"logits/rejected": -36353403.974842764,
|
||
|
|
"logps/chosen": -472.5767663043478,
|
||
|
|
"logps/rejected": -383.88600628930817,
|
||
|
|
"loss": 0.5685,
|
||
|
|
"loss/base_kto": 3.1965625286102295,
|
||
|
|
"metrics/advantage_var": 632.8568115234375,
|
||
|
|
"regularization/mmd": 1.351798176765442,
|
||
|
|
"regularization/rkhs_norm": 260.4620666503906,
|
||
|
|
"rewards/chosen": 0.5138472089115877,
|
||
|
|
"rewards/margins": 0.9225423098998199,
|
||
|
|
"rewards/rejected": -0.40869510098823214,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 7.022293567657471,
|
||
|
|
"kl": 15.356457710266113,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -34751122.39876352,
|
||
|
|
"logits/rejected": -36463200.25276461,
|
||
|
|
"logps/chosen": -499.6193972179289,
|
||
|
|
"logps/rejected": -386.7578248420221,
|
||
|
|
"loss": 0.5667,
|
||
|
|
"loss/base_kto": 3.2281863689422607,
|
||
|
|
"metrics/advantage_var": 516.4188842773438,
|
||
|
|
"regularization/mmd": 1.3055304288864136,
|
||
|
|
"regularization/rkhs_norm": 251.54727172851562,
|
||
|
|
"rewards/chosen": 0.5532462401589064,
|
||
|
|
"rewards/margins": 0.8311311830076302,
|
||
|
|
"rewards/rejected": -0.27788494284872384,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 9.221452713012695,
|
||
|
|
"kl": 15.334212303161621,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -34571923.10754414,
|
||
|
|
"logits/rejected": -35196793.96042618,
|
||
|
|
"logps/chosen": -491.7119783306581,
|
||
|
|
"logps/rejected": -353.80498477929984,
|
||
|
|
"loss": 0.5508,
|
||
|
|
"loss/base_kto": 3.180082082748413,
|
||
|
|
"metrics/advantage_var": 453.50323486328125,
|
||
|
|
"regularization/mmd": 1.2262661457061768,
|
||
|
|
"regularization/rkhs_norm": 236.27481079101562,
|
||
|
|
"rewards/chosen": 0.5832736044404595,
|
||
|
|
"rewards/margins": 0.8477262092016848,
|
||
|
|
"rewards/rejected": -0.2644526047612253,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 13.601996421813965,
|
||
|
|
"kl": 9.088701248168945,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -34697375.03645008,
|
||
|
|
"logits/rejected": -36160870.1633282,
|
||
|
|
"logps/chosen": -482.32611925515056,
|
||
|
|
"logps/rejected": -385.1588020030817,
|
||
|
|
"loss": 0.5606,
|
||
|
|
"loss/base_kto": 3.2661895751953125,
|
||
|
|
"metrics/advantage_var": 430.220947265625,
|
||
|
|
"regularization/mmd": 1.2183843851089478,
|
||
|
|
"regularization/rkhs_norm": 234.7561492919922,
|
||
|
|
"rewards/chosen": 0.4164625068096276,
|
||
|
|
"rewards/margins": 0.7796778078060524,
|
||
|
|
"rewards/rejected": -0.3632153009964248,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 10.456588745117188,
|
||
|
|
"kl": 9.765243530273438,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -33571011.98086125,
|
||
|
|
"logits/rejected": -34984230.81163859,
|
||
|
|
"logps/chosen": -480.03369218500796,
|
||
|
|
"logps/rejected": -383.59741098774884,
|
||
|
|
"loss": 0.5617,
|
||
|
|
"loss/base_kto": 3.257220506668091,
|
||
|
|
"metrics/advantage_var": 498.43560791015625,
|
||
|
|
"regularization/mmd": 1.236567497253418,
|
||
|
|
"regularization/rkhs_norm": 238.2596435546875,
|
||
|
|
"rewards/chosen": 0.4319639555765301,
|
||
|
|
"rewards/margins": 0.7968944166246972,
|
||
|
|
"rewards/rejected": -0.3649304610481671,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 13.861790657043457,
|
||
|
|
"kl": 9.847267150878906,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34855167.59557662,
|
||
|
|
"logits/rejected": -35080321.780525506,
|
||
|
|
"logps/chosen": -456.07523696682466,
|
||
|
|
"logps/rejected": -403.0278931607419,
|
||
|
|
"loss": 0.5655,
|
||
|
|
"loss/base_kto": 3.247133731842041,
|
||
|
|
"metrics/advantage_var": 668.1593627929688,
|
||
|
|
"regularization/mmd": 1.276819109916687,
|
||
|
|
"regularization/rkhs_norm": 246.0152587890625,
|
||
|
|
"rewards/chosen": 0.36525370762058157,
|
||
|
|
"rewards/margins": 0.7917951141724556,
|
||
|
|
"rewards/rejected": -0.42654140655187406,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 11.58325481414795,
|
||
|
|
"kl": 8.980515480041504,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -34816525.97110754,
|
||
|
|
"logits/rejected": -35051142.818873666,
|
||
|
|
"logps/chosen": -497.8179173354735,
|
||
|
|
"logps/rejected": -374.10816210045664,
|
||
|
|
"loss": 0.5605,
|
||
|
|
"loss/base_kto": 3.2262959480285645,
|
||
|
|
"metrics/advantage_var": 480.5451354980469,
|
||
|
|
"regularization/mmd": 1.2574070692062378,
|
||
|
|
"regularization/rkhs_norm": 242.2749481201172,
|
||
|
|
"rewards/chosen": 0.4310284609970656,
|
||
|
|
"rewards/margins": 0.8517019881267744,
|
||
|
|
"rewards/rejected": -0.4206735271297089,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 9.798060417175293,
|
||
|
|
"kl": 10.43753719329834,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -35005108.28169014,
|
||
|
|
"logits/rejected": -35699582.60218409,
|
||
|
|
"logps/chosen": -480.3617957746479,
|
||
|
|
"logps/rejected": -391.8496246099844,
|
||
|
|
"loss": 0.5508,
|
||
|
|
"loss/base_kto": 3.20361328125,
|
||
|
|
"metrics/advantage_var": 450.1048889160156,
|
||
|
|
"regularization/mmd": 1.2024950981140137,
|
||
|
|
"regularization/rkhs_norm": 231.6946258544922,
|
||
|
|
"rewards/chosen": 0.4042446751363214,
|
||
|
|
"rewards/margins": 0.8451452188729146,
|
||
|
|
"rewards/rejected": -0.4409005437365932,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 12.717971801757812,
|
||
|
|
"kl": 14.745546340942383,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -35810253.43209877,
|
||
|
|
"logits/rejected": -37534616.30379747,
|
||
|
|
"logps/chosen": -490.432918595679,
|
||
|
|
"logps/rejected": -366.3148734177215,
|
||
|
|
"loss": 0.5669,
|
||
|
|
"loss/base_kto": 3.1946423053741455,
|
||
|
|
"metrics/advantage_var": 550.33056640625,
|
||
|
|
"regularization/mmd": 1.34068763256073,
|
||
|
|
"regularization/rkhs_norm": 258.32135009765625,
|
||
|
|
"rewards/chosen": 0.5913930822301794,
|
||
|
|
"rewards/margins": 0.8842159562752496,
|
||
|
|
"rewards/rejected": -0.29282287404507024,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 13.095698356628418,
|
||
|
|
"kl": 8.906962394714355,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -33991900.23233909,
|
||
|
|
"logits/rejected": -35287810.78693624,
|
||
|
|
"logps/chosen": -498.6856848508634,
|
||
|
|
"logps/rejected": -382.642301710731,
|
||
|
|
"loss": 0.564,
|
||
|
|
"loss/base_kto": 3.180330753326416,
|
||
|
|
"metrics/advantage_var": 606.4514770507812,
|
||
|
|
"regularization/mmd": 1.3320293426513672,
|
||
|
|
"regularization/rkhs_norm": 256.6530456542969,
|
||
|
|
"rewards/chosen": 0.49957534095368916,
|
||
|
|
"rewards/margins": 0.9116029876962533,
|
||
|
|
"rewards/rejected": -0.41202764674256415,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 9.971153259277344,
|
||
|
|
"kl": 11.401013374328613,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34561693.17933131,
|
||
|
|
"logits/rejected": -36233870.03870968,
|
||
|
|
"logps/chosen": -488.33130699088144,
|
||
|
|
"logps/rejected": -350.17011088709677,
|
||
|
|
"loss": 0.5301,
|
||
|
|
"loss/base_kto": 3.1072757244110107,
|
||
|
|
"metrics/advantage_var": 364.1158752441406,
|
||
|
|
"regularization/mmd": 1.1331380605697632,
|
||
|
|
"regularization/rkhs_norm": 218.33103942871094,
|
||
|
|
"rewards/chosen": 0.5380707714695336,
|
||
|
|
"rewards/margins": 0.9163498360171647,
|
||
|
|
"rewards/rejected": -0.37827906454763105,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 10.389137268066406,
|
||
|
|
"kl": 14.117828369140625,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -33827361.267828844,
|
||
|
|
"logits/rejected": -34959454.66872111,
|
||
|
|
"logps/chosen": -464.130645800317,
|
||
|
|
"logps/rejected": -392.2838020030817,
|
||
|
|
"loss": 0.5229,
|
||
|
|
"loss/base_kto": 3.09826397895813,
|
||
|
|
"metrics/advantage_var": 331.5031433105469,
|
||
|
|
"regularization/mmd": 1.084905982017517,
|
||
|
|
"regularization/rkhs_norm": 209.0377655029297,
|
||
|
|
"rewards/chosen": 0.5963450097811014,
|
||
|
|
"rewards/margins": 0.9037712625723547,
|
||
|
|
"rewards/rejected": -0.30742625279125335,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 8.604015350341797,
|
||
|
|
"kl": 8.978219032287598,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -34979393.602356404,
|
||
|
|
"logits/rejected": -36312972.13976706,
|
||
|
|
"logps/chosen": -488.62638070692196,
|
||
|
|
"logps/rejected": -365.0787229617305,
|
||
|
|
"loss": 0.5302,
|
||
|
|
"loss/base_kto": 3.0801150798797607,
|
||
|
|
"metrics/advantage_var": 380.2249755859375,
|
||
|
|
"regularization/mmd": 1.16116464138031,
|
||
|
|
"regularization/rkhs_norm": 223.7311553955078,
|
||
|
|
"rewards/chosen": 0.5334298719538153,
|
||
|
|
"rewards/margins": 0.9683087115830791,
|
||
|
|
"rewards/rejected": -0.4348788396292637,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 10.985217094421387,
|
||
|
|
"kl": 14.511804580688477,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -36486346.383480825,
|
||
|
|
"logits/rejected": -36747512.34551495,
|
||
|
|
"logps/chosen": -495.9096607669617,
|
||
|
|
"logps/rejected": -396.19160091362124,
|
||
|
|
"loss": 0.5287,
|
||
|
|
"loss/base_kto": 3.0640370845794678,
|
||
|
|
"metrics/advantage_var": 381.7714538574219,
|
||
|
|
"regularization/mmd": 1.1654428243637085,
|
||
|
|
"regularization/rkhs_norm": 224.55545043945312,
|
||
|
|
"rewards/chosen": 0.6548410151208748,
|
||
|
|
"rewards/margins": 0.9437970584529969,
|
||
|
|
"rewards/rejected": -0.2889560433321221,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 10.690685272216797,
|
||
|
|
"kl": 16.309289932250977,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -35286133.915151514,
|
||
|
|
"logits/rejected": -36748786.787096776,
|
||
|
|
"logps/chosen": -483.8376893939394,
|
||
|
|
"logps/rejected": -386.22842741935483,
|
||
|
|
"loss": 0.5286,
|
||
|
|
"loss/base_kto": 3.0449044704437256,
|
||
|
|
"metrics/advantage_var": 424.19482421875,
|
||
|
|
"regularization/mmd": 1.1837689876556396,
|
||
|
|
"regularization/rkhs_norm": 228.0865020751953,
|
||
|
|
"rewards/chosen": 0.6678506562204072,
|
||
|
|
"rewards/margins": 0.9900833025472722,
|
||
|
|
"rewards/rejected": -0.32223264632686494,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 9.810893058776855,
|
||
|
|
"kl": 14.84647274017334,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -35376230.702064894,
|
||
|
|
"logits/rejected": -36343113.99335548,
|
||
|
|
"logps/chosen": -495.2001290560472,
|
||
|
|
"logps/rejected": -388.9727990033223,
|
||
|
|
"loss": 0.5315,
|
||
|
|
"loss/base_kto": 3.0942509174346924,
|
||
|
|
"metrics/advantage_var": 382.7725524902344,
|
||
|
|
"regularization/mmd": 1.1580281257629395,
|
||
|
|
"regularization/rkhs_norm": 223.12681579589844,
|
||
|
|
"rewards/chosen": 0.649119419334209,
|
||
|
|
"rewards/margins": 0.920761857139182,
|
||
|
|
"rewards/rejected": -0.271642437804973,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 10.877507209777832,
|
||
|
|
"kl": 17.48152732849121,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -35132161.23274478,
|
||
|
|
"logits/rejected": -35961845.08980213,
|
||
|
|
"logps/chosen": -474.65048154093097,
|
||
|
|
"logps/rejected": -376.8831811263318,
|
||
|
|
"loss": 0.5274,
|
||
|
|
"loss/base_kto": 3.0635948181152344,
|
||
|
|
"metrics/advantage_var": 374.0828552246094,
|
||
|
|
"regularization/mmd": 1.1555202007293701,
|
||
|
|
"regularization/rkhs_norm": 222.64358520507812,
|
||
|
|
"rewards/chosen": 0.6907168368466593,
|
||
|
|
"rewards/margins": 0.954688615565326,
|
||
|
|
"rewards/rejected": -0.2639717787186668,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 9.450188636779785,
|
||
|
|
"kl": 10.73950481414795,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34716625.816485226,
|
||
|
|
"logits/rejected": -35182775.25902669,
|
||
|
|
"logps/chosen": -488.67282270606535,
|
||
|
|
"logps/rejected": -349.94608516483515,
|
||
|
|
"loss": 0.5252,
|
||
|
|
"loss/base_kto": 3.1477468013763428,
|
||
|
|
"metrics/advantage_var": 330.1083068847656,
|
||
|
|
"regularization/mmd": 1.0534805059432983,
|
||
|
|
"regularization/rkhs_norm": 202.98275756835938,
|
||
|
|
"rewards/chosen": 0.4984940548150758,
|
||
|
|
"rewards/margins": 0.8629275949077956,
|
||
|
|
"rewards/rejected": -0.3644335400927198,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 9.99582290649414,
|
||
|
|
"kl": 15.615806579589844,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -34721485.4522293,
|
||
|
|
"logits/rejected": -35511142.08588957,
|
||
|
|
"logps/chosen": -442.3390724522293,
|
||
|
|
"logps/rejected": -392.20252588190186,
|
||
|
|
"loss": 0.524,
|
||
|
|
"loss/base_kto": 3.085685968399048,
|
||
|
|
"metrics/advantage_var": 377.5616149902344,
|
||
|
|
"regularization/mmd": 1.1066991090774536,
|
||
|
|
"regularization/rkhs_norm": 213.23684692382812,
|
||
|
|
"rewards/chosen": 0.554456479989799,
|
||
|
|
"rewards/margins": 0.928293910042761,
|
||
|
|
"rewards/rejected": -0.37383743005296205,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 8.671745300292969,
|
||
|
|
"kl": 11.820478439331055,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -34835478.61198738,
|
||
|
|
"logits/rejected": -35504559.15789474,
|
||
|
|
"logps/chosen": -467.85484029968455,
|
||
|
|
"logps/rejected": -375.56946594427245,
|
||
|
|
"loss": 0.5259,
|
||
|
|
"loss/base_kto": 3.0810964107513428,
|
||
|
|
"metrics/advantage_var": 367.2216491699219,
|
||
|
|
"regularization/mmd": 1.1258373260498047,
|
||
|
|
"regularization/rkhs_norm": 216.92431640625,
|
||
|
|
"rewards/chosen": 0.5543758259962785,
|
||
|
|
"rewards/margins": 0.9589037086697015,
|
||
|
|
"rewards/rejected": -0.404527882673423,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 9.208166122436523,
|
||
|
|
"kl": 12.190825462341309,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -35194234.649517685,
|
||
|
|
"logits/rejected": -36754758.80851064,
|
||
|
|
"logps/chosen": -494.23728898713824,
|
||
|
|
"logps/rejected": -358.1865738981763,
|
||
|
|
"loss": 0.5276,
|
||
|
|
"loss/base_kto": 3.098057985305786,
|
||
|
|
"metrics/advantage_var": 362.88330078125,
|
||
|
|
"regularization/mmd": 1.1229840517044067,
|
||
|
|
"regularization/rkhs_norm": 216.3745574951172,
|
||
|
|
"rewards/chosen": 0.5411652690727994,
|
||
|
|
"rewards/margins": 0.9349086275573169,
|
||
|
|
"rewards/rejected": -0.39374335848451747,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 11.298110961914062,
|
||
|
|
"kl": 11.307663917541504,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35621310.72196621,
|
||
|
|
"logits/rejected": -36497704.292527825,
|
||
|
|
"logps/chosen": -459.1538498463902,
|
||
|
|
"logps/rejected": -371.0775288155803,
|
||
|
|
"loss": 0.5246,
|
||
|
|
"loss/base_kto": 3.0882489681243896,
|
||
|
|
"metrics/advantage_var": 366.67852783203125,
|
||
|
|
"regularization/mmd": 1.108572006225586,
|
||
|
|
"regularization/rkhs_norm": 213.5977020263672,
|
||
|
|
"rewards/chosen": 0.5571527202740976,
|
||
|
|
"rewards/margins": 0.9439781605035044,
|
||
|
|
"rewards/rejected": -0.3868254402294068,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 8.899885177612305,
|
||
|
|
"kl": 14.008285522460938,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -34340141.74294671,
|
||
|
|
"logits/rejected": -34434909.30841122,
|
||
|
|
"logps/chosen": -480.7867358934169,
|
||
|
|
"logps/rejected": -356.93443341121497,
|
||
|
|
"loss": 0.5305,
|
||
|
|
"loss/base_kto": 3.1063425540924072,
|
||
|
|
"metrics/advantage_var": 375.7760314941406,
|
||
|
|
"regularization/mmd": 1.1377207040786743,
|
||
|
|
"regularization/rkhs_norm": 219.21401977539062,
|
||
|
|
"rewards/chosen": 0.586709432093701,
|
||
|
|
"rewards/margins": 0.9054192601228701,
|
||
|
|
"rewards/rejected": -0.3187098280291691,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 11.797314643859863,
|
||
|
|
"kl": 12.688746452331543,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -33929174.97688752,
|
||
|
|
"logits/rejected": -37030371.60063391,
|
||
|
|
"logps/chosen": -512.4694722650231,
|
||
|
|
"logps/rejected": -380.10239203645006,
|
||
|
|
"loss": 0.5319,
|
||
|
|
"loss/base_kto": 3.0777459144592285,
|
||
|
|
"metrics/advantage_var": 405.9903869628906,
|
||
|
|
"regularization/mmd": 1.1776024103164673,
|
||
|
|
"regularization/rkhs_norm": 226.89834594726562,
|
||
|
|
"rewards/chosen": 0.5939235128864118,
|
||
|
|
"rewards/margins": 0.9247837777809245,
|
||
|
|
"rewards/rejected": -0.3308602648945127,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 9.855902671813965,
|
||
|
|
"kl": 14.851201057434082,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -34853804.408163264,
|
||
|
|
"logits/rejected": -36198015.26011561,
|
||
|
|
"logps/chosen": -487.37744472789115,
|
||
|
|
"logps/rejected": -391.79588150289015,
|
||
|
|
"loss": 0.5265,
|
||
|
|
"loss/base_kto": 3.0706169605255127,
|
||
|
|
"metrics/advantage_var": 392.9869689941406,
|
||
|
|
"regularization/mmd": 1.1414614915847778,
|
||
|
|
"regularization/rkhs_norm": 219.93479919433594,
|
||
|
|
"rewards/chosen": 0.5632651192801339,
|
||
|
|
"rewards/margins": 0.9373461079932164,
|
||
|
|
"rewards/rejected": -0.37408098871308254,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 9.871848106384277,
|
||
|
|
"kl": 8.795090675354004,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -34945181.2864,
|
||
|
|
"logits/rejected": -35831795.49312977,
|
||
|
|
"logps/chosen": -500.3369,
|
||
|
|
"logps/rejected": -381.2418893129771,
|
||
|
|
"loss": 0.5279,
|
||
|
|
"loss/base_kto": 3.097670316696167,
|
||
|
|
"metrics/advantage_var": 380.62872314453125,
|
||
|
|
"regularization/mmd": 1.1255066394805908,
|
||
|
|
"regularization/rkhs_norm": 216.86062622070312,
|
||
|
|
"rewards/chosen": 0.47675625,
|
||
|
|
"rewards/margins": 0.9335280772900763,
|
||
|
|
"rewards/rejected": -0.4567718272900763,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 8.385743141174316,
|
||
|
|
"kl": 8.921009063720703,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -34484803.434146345,
|
||
|
|
"logits/rejected": -36060999.218045115,
|
||
|
|
"logps/chosen": -492.82560975609755,
|
||
|
|
"logps/rejected": -349.38188439849625,
|
||
|
|
"loss": 0.5237,
|
||
|
|
"loss/base_kto": 3.0965182781219482,
|
||
|
|
"metrics/advantage_var": 345.7809753417969,
|
||
|
|
"regularization/mmd": 1.0931130647659302,
|
||
|
|
"regularization/rkhs_norm": 210.6190948486328,
|
||
|
|
"rewards/chosen": 0.4762144507431402,
|
||
|
|
"rewards/margins": 0.9288405227591177,
|
||
|
|
"rewards/rejected": -0.45262607201597743,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 9.640080451965332,
|
||
|
|
"kl": 9.840849876403809,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -34486997.793528505,
|
||
|
|
"logits/rejected": -36593251.803486526,
|
||
|
|
"logps/chosen": -475.4094761171032,
|
||
|
|
"logps/rejected": -364.7089936608558,
|
||
|
|
"loss": 0.5308,
|
||
|
|
"loss/base_kto": 3.096221685409546,
|
||
|
|
"metrics/advantage_var": 378.3948669433594,
|
||
|
|
"regularization/mmd": 1.1503757238388062,
|
||
|
|
"regularization/rkhs_norm": 221.65237426757812,
|
||
|
|
"rewards/chosen": 0.5039539307769164,
|
||
|
|
"rewards/margins": 0.9324591285346917,
|
||
|
|
"rewards/rejected": -0.42850519775777535,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 11.856900215148926,
|
||
|
|
"kl": 11.206775665283203,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -34000229.34897361,
|
||
|
|
"logits/rejected": -35028485.13712375,
|
||
|
|
"logps/chosen": -479.1497434017595,
|
||
|
|
"logps/rejected": -356.11266722408027,
|
||
|
|
"loss": 0.5258,
|
||
|
|
"loss/base_kto": 3.1185264587402344,
|
||
|
|
"metrics/advantage_var": 332.6591796875,
|
||
|
|
"regularization/mmd": 1.0881156921386719,
|
||
|
|
"regularization/rkhs_norm": 209.65623474121094,
|
||
|
|
"rewards/chosen": 0.5253451171159045,
|
||
|
|
"rewards/margins": 0.9024306720744903,
|
||
|
|
"rewards/rejected": -0.3770855549585859,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 11.317198753356934,
|
||
|
|
"kl": 14.708724975585938,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -33627358.072289154,
|
||
|
|
"logits/rejected": -34988776.72727273,
|
||
|
|
"logps/chosen": -487.5407567771084,
|
||
|
|
"logps/rejected": -379.77227069805195,
|
||
|
|
"loss": 0.5208,
|
||
|
|
"loss/base_kto": 3.035454273223877,
|
||
|
|
"metrics/advantage_var": 356.38140869140625,
|
||
|
|
"regularization/mmd": 1.1312255859375,
|
||
|
|
"regularization/rkhs_norm": 217.96255493164062,
|
||
|
|
"rewards/chosen": 0.6100821437605892,
|
||
|
|
"rewards/margins": 0.985807951842278,
|
||
|
|
"rewards/rejected": -0.3757258080816888,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 9.777588844299316,
|
||
|
|
"kl": 15.399670600891113,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35688000.0,
|
||
|
|
"logits/rejected": -36090995.809523806,
|
||
|
|
"logps/chosen": -473.5448704769737,
|
||
|
|
"logps/rejected": -372.62806919642856,
|
||
|
|
"loss": 0.5246,
|
||
|
|
"loss/base_kto": 3.0967891216278076,
|
||
|
|
"metrics/advantage_var": 350.7513427734375,
|
||
|
|
"regularization/mmd": 1.100196361541748,
|
||
|
|
"regularization/rkhs_norm": 211.98388671875,
|
||
|
|
"rewards/chosen": 0.5481959895083779,
|
||
|
|
"rewards/margins": 0.898092942727837,
|
||
|
|
"rewards/rejected": -0.3498969532194592,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 11.24364948272705,
|
||
|
|
"kl": 13.601666450500488,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -34805155.41267388,
|
||
|
|
"logits/rejected": -35978830.45813586,
|
||
|
|
"logps/chosen": -472.4669629057187,
|
||
|
|
"logps/rejected": -390.4728475513428,
|
||
|
|
"loss": 0.519,
|
||
|
|
"loss/base_kto": 3.0523951053619385,
|
||
|
|
"metrics/advantage_var": 336.05792236328125,
|
||
|
|
"regularization/mmd": 1.0994646549224854,
|
||
|
|
"regularization/rkhs_norm": 211.84292602539062,
|
||
|
|
"rewards/chosen": 0.5824890608396928,
|
||
|
|
"rewards/margins": 0.9463016204285183,
|
||
|
|
"rewards/rejected": -0.3638125595888255,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 11.328319549560547,
|
||
|
|
"kl": 13.101523399353027,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -35008784.852664575,
|
||
|
|
"logits/rejected": -35833434.91588785,
|
||
|
|
"logps/chosen": -484.2189459247649,
|
||
|
|
"logps/rejected": -367.42481989875387,
|
||
|
|
"loss": 0.5277,
|
||
|
|
"loss/base_kto": 3.097578287124634,
|
||
|
|
"metrics/advantage_var": 360.671875,
|
||
|
|
"regularization/mmd": 1.1240640878677368,
|
||
|
|
"regularization/rkhs_norm": 216.58267211914062,
|
||
|
|
"rewards/chosen": 0.5724129034060296,
|
||
|
|
"rewards/margins": 0.9307274413337701,
|
||
|
|
"rewards/rejected": -0.3583145379277405,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 11.297466278076172,
|
||
|
|
"kl": 13.6049165725708,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -34877835.341772154,
|
||
|
|
"logits/rejected": -36181497.67901234,
|
||
|
|
"logps/chosen": -489.19274129746833,
|
||
|
|
"logps/rejected": -374.00308641975306,
|
||
|
|
"loss": 0.5267,
|
||
|
|
"loss/base_kto": 3.060823917388916,
|
||
|
|
"metrics/advantage_var": 387.2250671386719,
|
||
|
|
"regularization/mmd": 1.1527303457260132,
|
||
|
|
"regularization/rkhs_norm": 222.1060333251953,
|
||
|
|
"rewards/chosen": 0.5754118327853046,
|
||
|
|
"rewards/margins": 0.9552387434871035,
|
||
|
|
"rewards/rejected": -0.3798269107017988,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 8.630786895751953,
|
||
|
|
"kl": 10.696185111999512,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -35819656.53333333,
|
||
|
|
"logits/rejected": -35830756.58582677,
|
||
|
|
"logps/chosen": -489.590019379845,
|
||
|
|
"logps/rejected": -369.027312992126,
|
||
|
|
"loss": 0.5269,
|
||
|
|
"loss/base_kto": 3.0811078548431396,
|
||
|
|
"metrics/advantage_var": 372.056640625,
|
||
|
|
"regularization/mmd": 1.133884072303772,
|
||
|
|
"regularization/rkhs_norm": 218.47476196289062,
|
||
|
|
"rewards/chosen": 0.5522438226744186,
|
||
|
|
"rewards/margins": 0.9497557990216134,
|
||
|
|
"rewards/rejected": -0.3975119763471949,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 8.296945571899414,
|
||
|
|
"kl": 12.443785667419434,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -32976371.712,
|
||
|
|
"logits/rejected": -35256835.908396944,
|
||
|
|
"logps/chosen": -450.6808,
|
||
|
|
"logps/rejected": -388.9175095419847,
|
||
|
|
"loss": 0.5257,
|
||
|
|
"loss/base_kto": 3.0977537631988525,
|
||
|
|
"metrics/advantage_var": 346.5660095214844,
|
||
|
|
"regularization/mmd": 1.108106255531311,
|
||
|
|
"regularization/rkhs_norm": 213.5079345703125,
|
||
|
|
"rewards/chosen": 0.540194091796875,
|
||
|
|
"rewards/margins": 0.9232885649004056,
|
||
|
|
"rewards/rejected": -0.3830944731035305,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 13.444147109985352,
|
||
|
|
"kl": 11.908686637878418,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -34839270.07594936,
|
||
|
|
"logits/rejected": -36160881.777777776,
|
||
|
|
"logps/chosen": -458.37722507911394,
|
||
|
|
"logps/rejected": -365.0754484953704,
|
||
|
|
"loss": 0.5305,
|
||
|
|
"loss/base_kto": 3.1089224815368652,
|
||
|
|
"metrics/advantage_var": 379.64141845703125,
|
||
|
|
"regularization/mmd": 1.1348615884780884,
|
||
|
|
"regularization/rkhs_norm": 218.6631317138672,
|
||
|
|
"rewards/chosen": 0.5366746926609474,
|
||
|
|
"rewards/margins": 0.9034156847007573,
|
||
|
|
"rewards/rejected": -0.36674099203981,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 12.177067756652832,
|
||
|
|
"kl": 11.225503921508789,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -34948154.426426426,
|
||
|
|
"logits/rejected": -36317157.315960914,
|
||
|
|
"logps/chosen": -502.1341966966967,
|
||
|
|
"logps/rejected": -374.78242060260584,
|
||
|
|
"loss": 0.5313,
|
||
|
|
"loss/base_kto": 3.0800302028656006,
|
||
|
|
"metrics/advantage_var": 410.7743225097656,
|
||
|
|
"regularization/mmd": 1.1706825494766235,
|
||
|
|
"regularization/rkhs_norm": 225.56504821777344,
|
||
|
|
"rewards/chosen": 0.5339437032247091,
|
||
|
|
"rewards/margins": 0.9532965794716552,
|
||
|
|
"rewards/rejected": -0.4193528762469462,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 10.157988548278809,
|
||
|
|
"kl": 11.286453247070312,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -33943850.93167702,
|
||
|
|
"logits/rejected": -34867805.3836478,
|
||
|
|
"logps/chosen": -474.1967682453416,
|
||
|
|
"logps/rejected": -384.49513561320754,
|
||
|
|
"loss": 0.5288,
|
||
|
|
"loss/base_kto": 3.069972515106201,
|
||
|
|
"metrics/advantage_var": 385.8019714355469,
|
||
|
|
"regularization/mmd": 1.160033106803894,
|
||
|
|
"regularization/rkhs_norm": 223.51309204101562,
|
||
|
|
"rewards/chosen": 0.5911637774165373,
|
||
|
|
"rewards/margins": 0.9623554379796884,
|
||
|
|
"rewards/rejected": -0.37119166056315106,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 13.148500442504883,
|
||
|
|
"kl": 13.676080703735352,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -34414068.69400631,
|
||
|
|
"logits/rejected": -34114027.39318886,
|
||
|
|
"logps/chosen": -477.7570977917981,
|
||
|
|
"logps/rejected": -375.2875386996904,
|
||
|
|
"loss": 0.5235,
|
||
|
|
"loss/base_kto": 3.0857295989990234,
|
||
|
|
"metrics/advantage_var": 341.5010986328125,
|
||
|
|
"regularization/mmd": 1.101922869682312,
|
||
|
|
"regularization/rkhs_norm": 212.3165283203125,
|
||
|
|
"rewards/chosen": 0.5339198067361248,
|
||
|
|
"rewards/margins": 0.9309395148558036,
|
||
|
|
"rewards/rejected": -0.3970197081196788,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 11.413169860839844,
|
||
|
|
"kl": 12.233805656433105,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34812983.088607594,
|
||
|
|
"logits/rejected": -36667815.506172836,
|
||
|
|
"logps/chosen": -478.06759295886076,
|
||
|
|
"logps/rejected": -386.8450520833333,
|
||
|
|
"loss": 0.5241,
|
||
|
|
"loss/base_kto": 3.1054160594940186,
|
||
|
|
"metrics/advantage_var": 347.2292175292969,
|
||
|
|
"regularization/mmd": 1.0871450901031494,
|
||
|
|
"regularization/rkhs_norm": 209.4691619873047,
|
||
|
|
"rewards/chosen": 0.5347468219225919,
|
||
|
|
"rewards/margins": 0.8970902686306118,
|
||
|
|
"rewards/rejected": -0.3623434467080199,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 11.140260696411133,
|
||
|
|
"kl": 11.593470573425293,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -34999963.968102075,
|
||
|
|
"logits/rejected": -36366627.675344564,
|
||
|
|
"logps/chosen": -476.30193381180226,
|
||
|
|
"logps/rejected": -378.9831307427259,
|
||
|
|
"loss": 0.5313,
|
||
|
|
"loss/base_kto": 3.123239040374756,
|
||
|
|
"metrics/advantage_var": 376.97369384765625,
|
||
|
|
"regularization/mmd": 1.127297282218933,
|
||
|
|
"regularization/rkhs_norm": 217.2056121826172,
|
||
|
|
"rewards/chosen": 0.5285895193973036,
|
||
|
|
"rewards/margins": 0.9173786276970379,
|
||
|
|
"rewards/rejected": -0.3887891082997344,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 13.17670726776123,
|
||
|
|
"kl": 10.057597160339355,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -36366156.68202765,
|
||
|
|
"logits/rejected": -36326483.027027026,
|
||
|
|
"logps/chosen": -481.9315476190476,
|
||
|
|
"logps/rejected": -392.48842408585057,
|
||
|
|
"loss": 0.5218,
|
||
|
|
"loss/base_kto": 3.0502097606658936,
|
||
|
|
"metrics/advantage_var": 358.9496765136719,
|
||
|
|
"regularization/mmd": 1.12405526638031,
|
||
|
|
"regularization/rkhs_norm": 216.5809783935547,
|
||
|
|
"rewards/chosen": 0.5587905907228062,
|
||
|
|
"rewards/margins": 0.9702582717827725,
|
||
|
|
"rewards/rejected": -0.41146768105996623,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 12.34128189086914,
|
||
|
|
"kl": 10.098648071289062,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -36351302.80851064,
|
||
|
|
"logits/rejected": -35416436.06430868,
|
||
|
|
"logps/chosen": -502.5996390577508,
|
||
|
|
"logps/rejected": -383.8128265675241,
|
||
|
|
"loss": 0.5316,
|
||
|
|
"loss/base_kto": 3.0268125534057617,
|
||
|
|
"metrics/advantage_var": 463.24932861328125,
|
||
|
|
"regularization/mmd": 1.226233959197998,
|
||
|
|
"regularization/rkhs_norm": 236.2686004638672,
|
||
|
|
"rewards/chosen": 0.5856340866320764,
|
||
|
|
"rewards/margins": 1.003644246336156,
|
||
|
|
"rewards/rejected": -0.41801015970407956,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 8.02364730834961,
|
||
|
|
"kl": 11.608260154724121,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -35031314.14173228,
|
||
|
|
"logits/rejected": -35180702.75968992,
|
||
|
|
"logps/chosen": -462.6746062992126,
|
||
|
|
"logps/rejected": -389.1953003875969,
|
||
|
|
"loss": 0.5217,
|
||
|
|
"loss/base_kto": 3.049663782119751,
|
||
|
|
"metrics/advantage_var": 362.2906494140625,
|
||
|
|
"regularization/mmd": 1.1242762804031372,
|
||
|
|
"regularization/rkhs_norm": 216.62356567382812,
|
||
|
|
"rewards/chosen": 0.5438826913908711,
|
||
|
|
"rewards/margins": 0.9636309710511182,
|
||
|
|
"rewards/rejected": -0.4197482796602471,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 10.093438148498535,
|
||
|
|
"kl": 11.339247703552246,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -35537365.54146341,
|
||
|
|
"logits/rejected": -37229547.98195489,
|
||
|
|
"logps/chosen": -489.6883130081301,
|
||
|
|
"logps/rejected": -376.6608082706767,
|
||
|
|
"loss": 0.5223,
|
||
|
|
"loss/base_kto": 3.079643487930298,
|
||
|
|
"metrics/advantage_var": 330.7992858886719,
|
||
|
|
"regularization/mmd": 1.0988330841064453,
|
||
|
|
"regularization/rkhs_norm": 211.72119140625,
|
||
|
|
"rewards/chosen": 0.543333770007622,
|
||
|
|
"rewards/margins": 0.9218065327249622,
|
||
|
|
"rewards/rejected": -0.3784727627173402,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 9.906542778015137,
|
||
|
|
"kl": 11.895674705505371,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -33608088.96099844,
|
||
|
|
"logits/rejected": -35572397.87167449,
|
||
|
|
"logps/chosen": -480.40980889235567,
|
||
|
|
"logps/rejected": -365.56895539906105,
|
||
|
|
"loss": 0.5209,
|
||
|
|
"loss/base_kto": 3.057544469833374,
|
||
|
|
"metrics/advantage_var": 352.38409423828125,
|
||
|
|
"regularization/mmd": 1.1100084781646729,
|
||
|
|
"regularization/rkhs_norm": 213.8744659423828,
|
||
|
|
"rewards/chosen": 0.5721382700522865,
|
||
|
|
"rewards/margins": 0.9589256965426523,
|
||
|
|
"rewards/rejected": -0.3867874264903658,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 8.99142837524414,
|
||
|
|
"kl": 12.557846069335938,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -34284889.660855785,
|
||
|
|
"logits/rejected": -35557476.97996918,
|
||
|
|
"logps/chosen": -480.24014461172743,
|
||
|
|
"logps/rejected": -370.47154275808936,
|
||
|
|
"loss": 0.5307,
|
||
|
|
"loss/base_kto": 3.073073148727417,
|
||
|
|
"metrics/advantage_var": 423.8663024902344,
|
||
|
|
"regularization/mmd": 1.1724817752838135,
|
||
|
|
"regularization/rkhs_norm": 225.91172790527344,
|
||
|
|
"rewards/chosen": 0.5563522314685272,
|
||
|
|
"rewards/margins": 0.9417645093720326,
|
||
|
|
"rewards/rejected": -0.3854122779035054,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.978042558275912e+17,
|
||
|
|
"train_loss": 0.5577822983367867,
|
||
|
|
"train_runtime": 11092.8639,
|
||
|
|
"train_samples_per_second": 13.362,
|
||
|
|
"train_steps_per_second": 0.209
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.978042558275912e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|