10b963ccb1
Model: Gueule-d-ange/aup-fullft-kto_mmd-mmdrho5.19e-3_kr0.1-seed4 Source: Original Platform
2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 23.90775489807129,
|
|
"kl": 12.681447982788086,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35304941.26829268,
|
|
"logits/rejected": -36776152.615384616,
|
|
"logps/chosen": -498.387480945122,
|
|
"logps/rejected": -364.2709334935897,
|
|
"loss": 0.589,
|
|
"loss/base_kto": 3.8656768798828125,
|
|
"metrics/advantage_var": 206.61790466308594,
|
|
"regularization/mmd": 0.8460672497749329,
|
|
"regularization/rkhs_norm": 163.01873779296875,
|
|
"rewards/chosen": 0.2146739727113305,
|
|
"rewards/margins": 0.12779606424919734,
|
|
"rewards/rejected": 0.08687790846213317,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 16.661895751953125,
|
|
"kl": 9.479524612426758,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36529332.33690658,
|
|
"logits/rejected": -36652876.350877196,
|
|
"logps/chosen": -483.9310394333844,
|
|
"logps/rejected": -404.54291267942585,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.8932931423187256,
|
|
"metrics/advantage_var": 187.1262969970703,
|
|
"regularization/mmd": 0.8067044615745544,
|
|
"regularization/rkhs_norm": 155.4343719482422,
|
|
"rewards/chosen": 0.12601447580054198,
|
|
"rewards/margins": 0.10988777010155218,
|
|
"rewards/rejected": 0.016126705698989795,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 17.62555503845215,
|
|
"kl": 10.208549499511719,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35283677.89280245,
|
|
"logits/rejected": -36556519.09409888,
|
|
"logps/chosen": -493.9445348392037,
|
|
"logps/rejected": -388.88800837320576,
|
|
"loss": 0.5882,
|
|
"loss/base_kto": 3.854313611984253,
|
|
"metrics/advantage_var": 210.4778289794922,
|
|
"regularization/mmd": 0.8510956764221191,
|
|
"regularization/rkhs_norm": 163.9875946044922,
|
|
"rewards/chosen": 0.18612776050888927,
|
|
"rewards/margins": 0.1310663958406713,
|
|
"rewards/rejected": 0.055061364668217955,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 14.005806922912598,
|
|
"kl": 2.2349817752838135,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37167114.413559325,
|
|
"logits/rejected": -37620780.52173913,
|
|
"logps/chosen": -490.4091101694915,
|
|
"logps/rejected": -394.4764492753623,
|
|
"loss": 0.5817,
|
|
"loss/base_kto": 3.8326363563537598,
|
|
"metrics/advantage_var": 186.0223846435547,
|
|
"regularization/mmd": 0.8209611773490906,
|
|
"regularization/rkhs_norm": 158.18136596679688,
|
|
"rewards/chosen": -0.11016082763671875,
|
|
"rewards/margins": 0.14244287463201993,
|
|
"rewards/rejected": -0.2526037022687387,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 17.238176345825195,
|
|
"kl": 10.287137031555176,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -34916587.32919255,
|
|
"logits/rejected": -36685221.83647799,
|
|
"logps/chosen": -481.85025232919253,
|
|
"logps/rejected": -353.9864878144654,
|
|
"loss": 0.5817,
|
|
"loss/base_kto": 3.8582093715667725,
|
|
"metrics/advantage_var": 175.45828247070312,
|
|
"regularization/mmd": 0.7956187129020691,
|
|
"regularization/rkhs_norm": 153.2984161376953,
|
|
"rewards/chosen": 0.07971524896088594,
|
|
"rewards/margins": 0.12481347873502516,
|
|
"rewards/rejected": -0.045098229774139215,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 15.216425895690918,
|
|
"kl": 7.181277751922607,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36097753.55910543,
|
|
"logits/rejected": -36191335.33944954,
|
|
"logps/chosen": -470.92831469648564,
|
|
"logps/rejected": -376.39475821865443,
|
|
"loss": 0.5829,
|
|
"loss/base_kto": 3.8616745471954346,
|
|
"metrics/advantage_var": 181.6775360107422,
|
|
"regularization/mmd": 0.8012104034423828,
|
|
"regularization/rkhs_norm": 154.37579345703125,
|
|
"rewards/chosen": 0.07693188335187139,
|
|
"rewards/margins": 0.11716781689485209,
|
|
"rewards/rejected": -0.0402359335429807,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 17.482717514038086,
|
|
"kl": 10.225573539733887,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35949917.47242921,
|
|
"logits/rejected": -37559647.422003284,
|
|
"logps/chosen": -496.74236214605065,
|
|
"logps/rejected": -368.60129310344826,
|
|
"loss": 0.5811,
|
|
"loss/base_kto": 3.8098678588867188,
|
|
"metrics/advantage_var": 199.4989471435547,
|
|
"regularization/mmd": 0.8390861749649048,
|
|
"regularization/rkhs_norm": 161.67364501953125,
|
|
"rewards/chosen": 0.16720601521021564,
|
|
"rewards/margins": 0.18618105854617878,
|
|
"rewards/rejected": -0.018975043335963157,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 13.397759437561035,
|
|
"kl": 7.023360729217529,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35450873.43589743,
|
|
"logits/rejected": -35668174.048780486,
|
|
"logps/chosen": -477.0907451923077,
|
|
"logps/rejected": -349.8881002286585,
|
|
"loss": 0.5793,
|
|
"loss/base_kto": 3.7969701290130615,
|
|
"metrics/advantage_var": 199.3280029296875,
|
|
"regularization/mmd": 0.8371647000312805,
|
|
"regularization/rkhs_norm": 161.3034210205078,
|
|
"rewards/chosen": 0.10001216790614983,
|
|
"rewards/margins": 0.19661575917380536,
|
|
"rewards/rejected": -0.09660359126765554,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 14.562145233154297,
|
|
"kl": 5.361706733703613,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -34593796.777604975,
|
|
"logits/rejected": -35693000.540031396,
|
|
"logps/chosen": -454.53003499222393,
|
|
"logps/rejected": -360.58442896389323,
|
|
"loss": 0.579,
|
|
"loss/base_kto": 3.832873582839966,
|
|
"metrics/advantage_var": 180.25230407714844,
|
|
"regularization/mmd": 0.7988579273223877,
|
|
"regularization/rkhs_norm": 153.9225311279297,
|
|
"rewards/chosen": 0.04733409555295763,
|
|
"rewards/margins": 0.15071037930338688,
|
|
"rewards/rejected": -0.10337628375042926,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 16.442813873291016,
|
|
"kl": 10.120930671691895,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -34595783.20126783,
|
|
"logits/rejected": -37669536.14791988,
|
|
"logps/chosen": -497.20973652931855,
|
|
"logps/rejected": -373.6237962249615,
|
|
"loss": 0.5775,
|
|
"loss/base_kto": 3.753746509552002,
|
|
"metrics/advantage_var": 217.498291015625,
|
|
"regularization/mmd": 0.8664171099662781,
|
|
"regularization/rkhs_norm": 166.93972778320312,
|
|
"rewards/chosen": 0.15937588747632106,
|
|
"rewards/margins": 0.22018825717707763,
|
|
"rewards/rejected": -0.060812369700756576,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 17.87065887451172,
|
|
"kl": 5.803158283233643,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36220817.99684044,
|
|
"logits/rejected": -36433166.63987635,
|
|
"logps/chosen": -481.3564375987362,
|
|
"logps/rejected": -376.2250772797527,
|
|
"loss": 0.5888,
|
|
"loss/base_kto": 3.793182373046875,
|
|
"metrics/advantage_var": 248.9573974609375,
|
|
"regularization/mmd": 0.9168861508369446,
|
|
"regularization/rkhs_norm": 176.6640167236328,
|
|
"rewards/chosen": 0.030898312633448114,
|
|
"rewards/margins": 0.2135540126566924,
|
|
"rewards/rejected": -0.1826557000232443,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 15.884252548217773,
|
|
"kl": 12.579646110534668,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36456635.33643411,
|
|
"logits/rejected": -36561599.09291339,
|
|
"logps/chosen": -473.1209302325581,
|
|
"logps/rejected": -384.91274606299214,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.786097288131714,
|
|
"metrics/advantage_var": 239.2100067138672,
|
|
"regularization/mmd": 0.9171223044395447,
|
|
"regularization/rkhs_norm": 176.70948791503906,
|
|
"rewards/chosen": 0.18350818249606346,
|
|
"rewards/margins": 0.18441684338473532,
|
|
"rewards/rejected": -0.000908660888671875,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 18.382980346679688,
|
|
"kl": 11.036842346191406,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35736208.33234421,
|
|
"logits/rejected": -37681736.66006601,
|
|
"logps/chosen": -479.64938798219583,
|
|
"logps/rejected": -371.23999587458746,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.7316291332244873,
|
|
"metrics/advantage_var": 280.1012878417969,
|
|
"regularization/mmd": 0.9384465217590332,
|
|
"regularization/rkhs_norm": 180.8182373046875,
|
|
"rewards/chosen": 0.2972555188821402,
|
|
"rewards/margins": 0.26465956967769216,
|
|
"rewards/rejected": 0.032595949204448034,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 18.595134735107422,
|
|
"kl": 12.837692260742188,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37466318.96229261,
|
|
"logits/rejected": -39280646.63857374,
|
|
"logps/chosen": -491.5295060331825,
|
|
"logps/rejected": -355.3635028363047,
|
|
"loss": 0.5853,
|
|
"loss/base_kto": 3.7730443477630615,
|
|
"metrics/advantage_var": 241.9972686767578,
|
|
"regularization/mmd": 0.9096607565879822,
|
|
"regularization/rkhs_norm": 175.27183532714844,
|
|
"rewards/chosen": 0.2359304068494886,
|
|
"rewards/margins": 0.20733151634398694,
|
|
"rewards/rejected": 0.02859889050550167,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 19.06772232055664,
|
|
"kl": 11.678929328918457,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36305840.73301738,
|
|
"logits/rejected": -37630303.357032456,
|
|
"logps/chosen": -492.65501579778834,
|
|
"logps/rejected": -383.62688369397216,
|
|
"loss": 0.5858,
|
|
"loss/base_kto": 3.7701878547668457,
|
|
"metrics/advantage_var": 244.46861267089844,
|
|
"regularization/mmd": 0.9164788126945496,
|
|
"regularization/rkhs_norm": 176.5855255126953,
|
|
"rewards/chosen": 0.1869154292825274,
|
|
"rewards/margins": 0.2168836055096076,
|
|
"rewards/rejected": -0.029968176227080215,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 23.510053634643555,
|
|
"kl": 10.49164867401123,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37824399.95136778,
|
|
"logits/rejected": -38689067.627009645,
|
|
"logps/chosen": -495.4928761398176,
|
|
"logps/rejected": -359.66881028938906,
|
|
"loss": 0.5875,
|
|
"loss/base_kto": 3.80584979057312,
|
|
"metrics/advantage_var": 232.99180603027344,
|
|
"regularization/mmd": 0.8941518664360046,
|
|
"regularization/rkhs_norm": 172.28359985351562,
|
|
"rewards/chosen": 0.14421909413439163,
|
|
"rewards/margins": 0.1627761336221752,
|
|
"rewards/rejected": -0.018557039487783547,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.82607078552246,
|
|
"kl": 7.366345405578613,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35904137.444270015,
|
|
"logits/rejected": -37476915.75738725,
|
|
"logps/chosen": -476.3798567503925,
|
|
"logps/rejected": -363.42952954898914,
|
|
"loss": 0.5814,
|
|
"loss/base_kto": 3.760831594467163,
|
|
"metrics/advantage_var": 233.8565216064453,
|
|
"regularization/mmd": 0.8902547955513,
|
|
"regularization/rkhs_norm": 171.53271484375,
|
|
"rewards/chosen": 0.09115862284949286,
|
|
"rewards/margins": 0.22523825493374758,
|
|
"rewards/rejected": -0.13407963208425472,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 16.150705337524414,
|
|
"kl": 11.253143310546875,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36481020.84437596,
|
|
"logits/rejected": -37468559.21394612,
|
|
"logps/chosen": -486.3712442218798,
|
|
"logps/rejected": -386.98638074484944,
|
|
"loss": 0.5781,
|
|
"loss/base_kto": 3.739018201828003,
|
|
"metrics/advantage_var": 224.9400634765625,
|
|
"regularization/mmd": 0.885532557964325,
|
|
"regularization/rkhs_norm": 170.6228485107422,
|
|
"rewards/chosen": 0.19712246511309467,
|
|
"rewards/margins": 0.22026693318783885,
|
|
"rewards/rejected": -0.023144468074744174,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 16.826560974121094,
|
|
"kl": 11.830695152282715,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -34938873.80332829,
|
|
"logits/rejected": -38308968.21970921,
|
|
"logps/chosen": -502.84880862329805,
|
|
"logps/rejected": -372.40619951534734,
|
|
"loss": 0.5859,
|
|
"loss/base_kto": 3.7713570594787598,
|
|
"metrics/advantage_var": 268.22540283203125,
|
|
"regularization/mmd": 0.9160144925117493,
|
|
"regularization/rkhs_norm": 176.49607849121094,
|
|
"rewards/chosen": 0.21172615547584153,
|
|
"rewards/margins": 0.1934679605447373,
|
|
"rewards/rejected": 0.018258194931104227,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 16.222583770751953,
|
|
"kl": 10.096343994140625,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35715616.646967344,
|
|
"logits/rejected": -37105972.64678179,
|
|
"logps/chosen": -461.8372861586314,
|
|
"logps/rejected": -363.0498920722135,
|
|
"loss": 0.5796,
|
|
"loss/base_kto": 3.748002290725708,
|
|
"metrics/advantage_var": 225.37364196777344,
|
|
"regularization/mmd": 0.8887432217597961,
|
|
"regularization/rkhs_norm": 171.24147033691406,
|
|
"rewards/chosen": 0.1531903042949006,
|
|
"rewards/margins": 0.23799627046705157,
|
|
"rewards/rejected": -0.08480596617215096,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 18.901710510253906,
|
|
"kl": 7.529710292816162,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35298238.42364532,
|
|
"logits/rejected": -36855258.61102831,
|
|
"logps/chosen": -473.2080767651888,
|
|
"logps/rejected": -352.67206128912073,
|
|
"loss": 0.5767,
|
|
"loss/base_kto": 3.7175674438476562,
|
|
"metrics/advantage_var": 235.353515625,
|
|
"regularization/mmd": 0.8964071273803711,
|
|
"regularization/rkhs_norm": 172.71815490722656,
|
|
"rewards/chosen": 0.14702863019871204,
|
|
"rewards/margins": 0.2649733845590418,
|
|
"rewards/rejected": -0.11794475436032974,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 14.329428672790527,
|
|
"kl": 5.690273284912109,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36533533.47878788,
|
|
"logits/rejected": -37200050.37419355,
|
|
"logps/chosen": -509.5935606060606,
|
|
"logps/rejected": -372.37386592741933,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.7858269214630127,
|
|
"metrics/advantage_var": 239.9020538330078,
|
|
"regularization/mmd": 0.9174237251281738,
|
|
"regularization/rkhs_norm": 176.767578125,
|
|
"rewards/chosen": 0.057962741273822206,
|
|
"rewards/margins": 0.21008462355866342,
|
|
"rewards/rejected": -0.15212188228484122,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 17.99163055419922,
|
|
"kl": 6.039793491363525,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36726399.78559464,
|
|
"logits/rejected": -37461209.39385066,
|
|
"logps/chosen": -472.76350502512565,
|
|
"logps/rejected": -402.56927159590043,
|
|
"loss": 0.5755,
|
|
"loss/base_kto": 3.721526861190796,
|
|
"metrics/advantage_var": 254.74917602539062,
|
|
"regularization/mmd": 0.8826885223388672,
|
|
"regularization/rkhs_norm": 170.07485961914062,
|
|
"rewards/chosen": -0.007207811377955042,
|
|
"rewards/margins": 0.2573227558861829,
|
|
"rewards/rejected": -0.264530567264138,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 20.017250061035156,
|
|
"kl": 7.532881259918213,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36146347.98151001,
|
|
"logits/rejected": -36922759.09984152,
|
|
"logps/chosen": -496.83840523882895,
|
|
"logps/rejected": -372.50168383518223,
|
|
"loss": 0.5828,
|
|
"loss/base_kto": 3.754721164703369,
|
|
"metrics/advantage_var": 261.9855651855469,
|
|
"regularization/mmd": 0.9074088335037231,
|
|
"regularization/rkhs_norm": 174.8379364013672,
|
|
"rewards/chosen": 0.08191461078191208,
|
|
"rewards/margins": 0.22445522963003262,
|
|
"rewards/rejected": -0.14254061884812055,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 15.541690826416016,
|
|
"kl": 9.11847972869873,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -34632853.19205298,
|
|
"logits/rejected": -36152507.834319524,
|
|
"logps/chosen": -489.0806084437086,
|
|
"logps/rejected": -374.3659393491124,
|
|
"loss": 0.581,
|
|
"loss/base_kto": 3.6857070922851562,
|
|
"metrics/advantage_var": 268.4297790527344,
|
|
"regularization/mmd": 0.962521493434906,
|
|
"regularization/rkhs_norm": 185.45693969726562,
|
|
"rewards/chosen": 0.1476656174817622,
|
|
"rewards/margins": 0.3030050108717631,
|
|
"rewards/rejected": -0.1553393933900009,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 14.271484375,
|
|
"kl": 10.363009452819824,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36159265.927710846,
|
|
"logits/rejected": -36064482.077922076,
|
|
"logps/chosen": -508.9699736445783,
|
|
"logps/rejected": -377.15163352272725,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.725666046142578,
|
|
"metrics/advantage_var": 329.1117858886719,
|
|
"regularization/mmd": 1.0289491415023804,
|
|
"regularization/rkhs_norm": 198.25608825683594,
|
|
"rewards/chosen": 0.18816217169704208,
|
|
"rewards/margins": 0.26723369964876564,
|
|
"rewards/rejected": -0.07907152795172356,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 14.527031898498535,
|
|
"kl": 7.910184383392334,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36705883.980922095,
|
|
"logits/rejected": -36531236.17818741,
|
|
"logps/chosen": -497.47406597774244,
|
|
"logps/rejected": -370.1050787250384,
|
|
"loss": 0.5809,
|
|
"loss/base_kto": 3.6962249279022217,
|
|
"metrics/advantage_var": 271.1266784667969,
|
|
"regularization/mmd": 0.9510257840156555,
|
|
"regularization/rkhs_norm": 183.24197387695312,
|
|
"rewards/chosen": 0.09800809365956317,
|
|
"rewards/margins": 0.27047829190850115,
|
|
"rewards/rejected": -0.17247019824893794,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 16.18922233581543,
|
|
"kl": 7.231024265289307,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35402629.51196172,
|
|
"logits/rejected": -36064514.74425727,
|
|
"logps/chosen": -480.5859250398724,
|
|
"logps/rejected": -362.80015313935684,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.7060494422912598,
|
|
"metrics/advantage_var": 267.0411071777344,
|
|
"regularization/mmd": 0.964124858379364,
|
|
"regularization/rkhs_norm": 185.76588439941406,
|
|
"rewards/chosen": 0.03829733378579172,
|
|
"rewards/margins": 0.282168721775718,
|
|
"rewards/rejected": -0.2438713879899263,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 19.428863525390625,
|
|
"kl": 6.699652194976807,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -34408579.405320816,
|
|
"logits/rejected": -36810844.65522621,
|
|
"logps/chosen": -472.9701682316119,
|
|
"logps/rejected": -392.33014820592825,
|
|
"loss": 0.5821,
|
|
"loss/base_kto": 3.7921574115753174,
|
|
"metrics/advantage_var": 216.35459899902344,
|
|
"regularization/mmd": 0.8642802238464355,
|
|
"regularization/rkhs_norm": 166.52798461914062,
|
|
"rewards/chosen": 0.05617255746665322,
|
|
"rewards/margins": 0.18439785127332736,
|
|
"rewards/rejected": -0.12822529380667413,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 13.99908447265625,
|
|
"kl": 11.094145774841309,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37669820.8256,
|
|
"logits/rejected": -37664030.09465649,
|
|
"logps/chosen": -468.0696,
|
|
"logps/rejected": -397.8416030534351,
|
|
"loss": 0.5725,
|
|
"loss/base_kto": 3.6761891841888428,
|
|
"metrics/advantage_var": 231.89492797851562,
|
|
"regularization/mmd": 0.9035652279853821,
|
|
"regularization/rkhs_norm": 174.0973358154297,
|
|
"rewards/chosen": 0.1345001220703125,
|
|
"rewards/margins": 0.2822148912735568,
|
|
"rewards/rejected": -0.14771476920324428,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 12.809364318847656,
|
|
"kl": 8.228848457336426,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35409770.63209076,
|
|
"logits/rejected": -36674515.209653094,
|
|
"logps/chosen": -493.70765802269045,
|
|
"logps/rejected": -365.242411387632,
|
|
"loss": 0.5781,
|
|
"loss/base_kto": 3.7145614624023438,
|
|
"metrics/advantage_var": 234.45376586914062,
|
|
"regularization/mmd": 0.9103992581367493,
|
|
"regularization/rkhs_norm": 175.41412353515625,
|
|
"rewards/chosen": 0.11652515232079999,
|
|
"rewards/margins": 0.25433267330658316,
|
|
"rewards/rejected": -0.13780752098578314,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 18.693523406982422,
|
|
"kl": 9.805913925170898,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35212472.75840978,
|
|
"logits/rejected": -36863155.936102234,
|
|
"logps/chosen": -486.26748853211006,
|
|
"logps/rejected": -368.1652605830671,
|
|
"loss": 0.578,
|
|
"loss/base_kto": 3.7344930171966553,
|
|
"metrics/advantage_var": 215.81729125976562,
|
|
"regularization/mmd": 0.889321506023407,
|
|
"regularization/rkhs_norm": 171.35292053222656,
|
|
"rewards/chosen": 0.14162699381510416,
|
|
"rewards/margins": 0.2428915208647934,
|
|
"rewards/rejected": -0.10126452704968925,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 24.230606079101562,
|
|
"kl": 7.2070631980896,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36908811.27044025,
|
|
"logits/rejected": -35498960.298136644,
|
|
"logps/chosen": -476.51994889937106,
|
|
"logps/rejected": -384.64834045031057,
|
|
"loss": 0.5796,
|
|
"loss/base_kto": 3.7236766815185547,
|
|
"metrics/advantage_var": 243.0139617919922,
|
|
"regularization/mmd": 0.9129219055175781,
|
|
"regularization/rkhs_norm": 175.9001922607422,
|
|
"rewards/chosen": 0.07593924444426531,
|
|
"rewards/margins": 0.25153042300659545,
|
|
"rewards/rejected": -0.17559117856233017,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 19.190467834472656,
|
|
"kl": 7.582091808319092,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36191556.37267081,
|
|
"logits/rejected": -36882010.16352201,
|
|
"logps/chosen": -489.1235442546584,
|
|
"logps/rejected": -365.31318789308176,
|
|
"loss": 0.5826,
|
|
"loss/base_kto": 3.775066375732422,
|
|
"metrics/advantage_var": 229.5860137939453,
|
|
"regularization/mmd": 0.8857846260070801,
|
|
"regularization/rkhs_norm": 170.6714324951172,
|
|
"rewards/chosen": 0.13538101148901519,
|
|
"rewards/margins": 0.2081826254466422,
|
|
"rewards/rejected": -0.07280161395762702,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 20.92852210998535,
|
|
"kl": 13.803125381469727,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36878574.13953488,
|
|
"logits/rejected": -37196543.596850395,
|
|
"logps/chosen": -489.7798449612403,
|
|
"logps/rejected": -366.0732529527559,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.735741376876831,
|
|
"metrics/advantage_var": 250.020263671875,
|
|
"regularization/mmd": 0.9350200891494751,
|
|
"regularization/rkhs_norm": 180.15802001953125,
|
|
"rewards/chosen": 0.23140116846838663,
|
|
"rewards/margins": 0.24454379777641136,
|
|
"rewards/rejected": -0.01314262930802473,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 19.827178955078125,
|
|
"kl": 9.982133865356445,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -37341917.5623987,
|
|
"logits/rejected": -38152565.76772247,
|
|
"logps/chosen": -480.5630571312804,
|
|
"logps/rejected": -385.2698906485671,
|
|
"loss": 0.5793,
|
|
"loss/base_kto": 3.7122788429260254,
|
|
"metrics/advantage_var": 239.6793975830078,
|
|
"regularization/mmd": 0.9221993684768677,
|
|
"regularization/rkhs_norm": 177.687744140625,
|
|
"rewards/chosen": 0.12878181791382826,
|
|
"rewards/margins": 0.270680121283779,
|
|
"rewards/rejected": -0.14189830336995074,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 17.058692932128906,
|
|
"kl": 4.240655422210693,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36959863.14465409,
|
|
"logits/rejected": -36906018.98136646,
|
|
"logps/chosen": -471.84994103773585,
|
|
"logps/rejected": -367.23163334627327,
|
|
"loss": 0.5794,
|
|
"loss/base_kto": 3.724996566772461,
|
|
"metrics/advantage_var": 232.15652465820312,
|
|
"regularization/mmd": 0.9098402261734009,
|
|
"regularization/rkhs_norm": 175.30642700195312,
|
|
"rewards/chosen": -0.040978053830704596,
|
|
"rewards/margins": 0.2626346966788063,
|
|
"rewards/rejected": -0.30361275050951086,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 13.596495628356934,
|
|
"kl": 14.249995231628418,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36507934.41203008,
|
|
"logits/rejected": -37932126.90731707,
|
|
"logps/chosen": -503.81954887218046,
|
|
"logps/rejected": -360.6779217479675,
|
|
"loss": 0.5784,
|
|
"loss/base_kto": 3.680901050567627,
|
|
"metrics/advantage_var": 252.53746032714844,
|
|
"regularization/mmd": 0.9465328454971313,
|
|
"regularization/rkhs_norm": 182.37628173828125,
|
|
"rewards/chosen": 0.24967366842398966,
|
|
"rewards/margins": 0.28866869480308643,
|
|
"rewards/rejected": -0.0389950263790968,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 16.117652893066406,
|
|
"kl": 11.919455528259277,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -37659727.25696594,
|
|
"logits/rejected": -38779605.87341772,
|
|
"logps/chosen": -470.2393575851393,
|
|
"logps/rejected": -362.3188538370253,
|
|
"loss": 0.5694,
|
|
"loss/base_kto": 3.4670631885528564,
|
|
"metrics/advantage_var": 372.93902587890625,
|
|
"regularization/mmd": 1.0879830121994019,
|
|
"regularization/rkhs_norm": 209.63064575195312,
|
|
"rewards/chosen": 0.40930671809996616,
|
|
"rewards/margins": 0.5463803677357639,
|
|
"rewards/rejected": -0.13707364963579782,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 12.704465866088867,
|
|
"kl": 9.122496604919434,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36618425.87458746,
|
|
"logits/rejected": -38060457.400593475,
|
|
"logps/chosen": -471.3839727722772,
|
|
"logps/rejected": -366.02346068249255,
|
|
"loss": 0.5713,
|
|
"loss/base_kto": 3.233105182647705,
|
|
"metrics/advantage_var": 561.404541015625,
|
|
"regularization/mmd": 1.3372970819473267,
|
|
"regularization/rkhs_norm": 257.6680603027344,
|
|
"rewards/chosen": 0.3727112823587046,
|
|
"rewards/margins": 0.8635088289265366,
|
|
"rewards/rejected": -0.490797546567832,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 15.845054626464844,
|
|
"kl": 7.558243751525879,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34239751.684044234,
|
|
"logits/rejected": -36231050.88098918,
|
|
"logps/chosen": -464.8693226698262,
|
|
"logps/rejected": -381.7858867851623,
|
|
"loss": 0.5691,
|
|
"loss/base_kto": 3.2288079261779785,
|
|
"metrics/advantage_var": 574.4853515625,
|
|
"regularization/mmd": 1.3237676620483398,
|
|
"regularization/rkhs_norm": 255.0612335205078,
|
|
"rewards/chosen": 0.41367517263403436,
|
|
"rewards/margins": 0.8618098175094208,
|
|
"rewards/rejected": -0.4481346448753864,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 12.698341369628906,
|
|
"kl": 8.256165504455566,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36301010.05128205,
|
|
"logits/rejected": -38198771.512195125,
|
|
"logps/chosen": -487.06590544871796,
|
|
"logps/rejected": -379.7520960365854,
|
|
"loss": 0.5684,
|
|
"loss/base_kto": 3.219708204269409,
|
|
"metrics/advantage_var": 547.5316772460938,
|
|
"regularization/mmd": 1.327531099319458,
|
|
"regularization/rkhs_norm": 255.7863311767578,
|
|
"rewards/chosen": 0.41121551318046373,
|
|
"rewards/margins": 0.8806920734474701,
|
|
"rewards/rejected": -0.46947656026700646,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 11.37092399597168,
|
|
"kl": 8.794673919677734,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35675913.331220284,
|
|
"logits/rejected": -37025309.18952234,
|
|
"logps/chosen": -473.1105388272583,
|
|
"logps/rejected": -376.31399268104775,
|
|
"loss": 0.5706,
|
|
"loss/base_kto": 3.216496706008911,
|
|
"metrics/advantage_var": 592.7921752929688,
|
|
"regularization/mmd": 1.3485082387924194,
|
|
"regularization/rkhs_norm": 259.82818603515625,
|
|
"rewards/chosen": 0.37435173119304677,
|
|
"rewards/margins": 0.8877941365763288,
|
|
"rewards/rejected": -0.513442405383282,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 10.955135345458984,
|
|
"kl": 11.502577781677246,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34308980.658227846,
|
|
"logits/rejected": -35746401.97530864,
|
|
"logps/chosen": -456.9647943037975,
|
|
"logps/rejected": -369.28120177469134,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.21818470954895,
|
|
"metrics/advantage_var": 499.4261169433594,
|
|
"regularization/mmd": 1.2681556940078735,
|
|
"regularization/rkhs_norm": 244.3459930419922,
|
|
"rewards/chosen": 0.46269433709639535,
|
|
"rewards/margins": 0.8524536833872662,
|
|
"rewards/rejected": -0.38975934629087094,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 11.038398742675781,
|
|
"kl": 18.00704574584961,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34835183.12893983,
|
|
"logits/rejected": -34586880.87972508,
|
|
"logps/chosen": -463.0404727793696,
|
|
"logps/rejected": -374.49151632302403,
|
|
"loss": 0.5675,
|
|
"loss/base_kto": 3.201265811920166,
|
|
"metrics/advantage_var": 578.2403564453125,
|
|
"regularization/mmd": 1.3390430212020874,
|
|
"regularization/rkhs_norm": 258.00445556640625,
|
|
"rewards/chosen": 0.5857707463567783,
|
|
"rewards/margins": 0.8486113581849302,
|
|
"rewards/rejected": -0.2628406118281518,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 11.733174324035645,
|
|
"kl": 20.5081844329834,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34788497.35247209,
|
|
"logits/rejected": -36518286.3093415,
|
|
"logps/chosen": -468.3646331738437,
|
|
"logps/rejected": -350.9655436447167,
|
|
"loss": 0.5542,
|
|
"loss/base_kto": 3.2452073097229004,
|
|
"metrics/advantage_var": 411.1564025878906,
|
|
"regularization/mmd": 1.1881401538848877,
|
|
"regularization/rkhs_norm": 228.92874145507812,
|
|
"rewards/chosen": 0.5807296047180273,
|
|
"rewards/margins": 0.7610419089007955,
|
|
"rewards/rejected": -0.18031230418276822,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 15.177841186523438,
|
|
"kl": 12.759466171264648,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35313090.81761006,
|
|
"logits/rejected": -36495560.347826086,
|
|
"logps/chosen": -474.4255110062893,
|
|
"logps/rejected": -369.9761257763975,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.229013204574585,
|
|
"metrics/advantage_var": 521.7021484375,
|
|
"regularization/mmd": 1.28860342502594,
|
|
"regularization/rkhs_norm": 248.2858123779297,
|
|
"rewards/chosen": 0.4632223357194625,
|
|
"rewards/margins": 0.8284195114976797,
|
|
"rewards/rejected": -0.3651971757782172,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 9.456308364868164,
|
|
"kl": 14.029640197753906,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34754284.612055644,
|
|
"logits/rejected": -35375521.36492891,
|
|
"logps/chosen": -488.0619204018547,
|
|
"logps/rejected": -349.91869075829385,
|
|
"loss": 0.5709,
|
|
"loss/base_kto": 3.2024781703948975,
|
|
"metrics/advantage_var": 556.343017578125,
|
|
"regularization/mmd": 1.3645777702331543,
|
|
"regularization/rkhs_norm": 262.9244079589844,
|
|
"rewards/chosen": 0.5272589422635723,
|
|
"rewards/margins": 0.8624981055824654,
|
|
"rewards/rejected": -0.33523916331889314,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 12.763233184814453,
|
|
"kl": 13.030585289001465,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35205675.754122935,
|
|
"logits/rejected": -35327893.08972268,
|
|
"logps/chosen": -498.0701836581709,
|
|
"logps/rejected": -373.61964722675367,
|
|
"loss": 0.5664,
|
|
"loss/base_kto": 3.2134625911712646,
|
|
"metrics/advantage_var": 646.94775390625,
|
|
"regularization/mmd": 1.317788004875183,
|
|
"regularization/rkhs_norm": 253.9090576171875,
|
|
"rewards/chosen": 0.5833422705151331,
|
|
"rewards/margins": 0.887209255892185,
|
|
"rewards/rejected": -0.3038669853770519,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 18.62638282775879,
|
|
"kl": 13.3413724899292,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35348151.57639939,
|
|
"logits/rejected": -37456331.8901454,
|
|
"logps/chosen": -500.2219648260212,
|
|
"logps/rejected": -376.20100464458807,
|
|
"loss": 0.5657,
|
|
"loss/base_kto": 3.220020294189453,
|
|
"metrics/advantage_var": 549.8190307617188,
|
|
"regularization/mmd": 1.3052831888198853,
|
|
"regularization/rkhs_norm": 251.4996337890625,
|
|
"rewards/chosen": 0.528188731414287,
|
|
"rewards/margins": 0.8517421834207364,
|
|
"rewards/rejected": -0.3235534520064494,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 15.56907844543457,
|
|
"kl": 17.59682273864746,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35049228.34726688,
|
|
"logits/rejected": -36515973.835866265,
|
|
"logps/chosen": -457.3328476688103,
|
|
"logps/rejected": -377.69125189969606,
|
|
"loss": 0.5639,
|
|
"loss/base_kto": 3.2209317684173584,
|
|
"metrics/advantage_var": 563.7723999023438,
|
|
"regularization/mmd": 1.2898712158203125,
|
|
"regularization/rkhs_norm": 248.5301055908203,
|
|
"rewards/chosen": 0.49676317417353294,
|
|
"rewards/margins": 0.8339053764407014,
|
|
"rewards/rejected": -0.3371422022671685,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 14.420731544494629,
|
|
"kl": 15.833075523376465,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35450782.32114468,
|
|
"logits/rejected": -36681284.42396314,
|
|
"logps/chosen": -489.96363275039744,
|
|
"logps/rejected": -383.83885368663596,
|
|
"loss": 0.5722,
|
|
"loss/base_kto": 3.184199094772339,
|
|
"metrics/advantage_var": 710.5199584960938,
|
|
"regularization/mmd": 1.3931556940078735,
|
|
"regularization/rkhs_norm": 268.4307556152344,
|
|
"rewards/chosen": 0.5141282801787311,
|
|
"rewards/margins": 0.8975747582533279,
|
|
"rewards/rejected": -0.38344647807459675,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.51924991607666,
|
|
"kl": 7.321361064910889,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -33592987.5443038,
|
|
"logits/rejected": -35911743.209876545,
|
|
"logps/chosen": -478.610957278481,
|
|
"logps/rejected": -397.3753858024691,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.250958204269409,
|
|
"metrics/advantage_var": 468.6973571777344,
|
|
"regularization/mmd": 1.237536907196045,
|
|
"regularization/rkhs_norm": 238.4464111328125,
|
|
"rewards/chosen": 0.3320492611655706,
|
|
"rewards/margins": 0.8196819184403584,
|
|
"rewards/rejected": -0.48763265727478783,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 10.645336151123047,
|
|
"kl": 13.960247993469238,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35695479.36431784,
|
|
"logits/rejected": -36218365.494290374,
|
|
"logps/chosen": -482.2657421289355,
|
|
"logps/rejected": -375.55189641109297,
|
|
"loss": 0.5679,
|
|
"loss/base_kto": 3.1963846683502197,
|
|
"metrics/advantage_var": 570.2843017578125,
|
|
"regularization/mmd": 1.3467546701431274,
|
|
"regularization/rkhs_norm": 259.4903259277344,
|
|
"rewards/chosen": 0.5581533269010026,
|
|
"rewards/margins": 0.8575564568395732,
|
|
"rewards/rejected": -0.29940312993857054,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 13.598370552062988,
|
|
"kl": 20.37125587463379,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -37020278.15384615,
|
|
"logits/rejected": -37173443.83792545,
|
|
"logps/chosen": -462.88353129713425,
|
|
"logps/rejected": -382.0808093598055,
|
|
"loss": 0.5577,
|
|
"loss/base_kto": 3.15634822845459,
|
|
"metrics/advantage_var": 540.4674072265625,
|
|
"regularization/mmd": 1.3054908514022827,
|
|
"regularization/rkhs_norm": 251.53965759277344,
|
|
"rewards/chosen": 0.6769872236755279,
|
|
"rewards/margins": 0.891834312308467,
|
|
"rewards/rejected": -0.2148470886329391,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 14.927032470703125,
|
|
"kl": 12.652097702026367,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35232756.8174727,
|
|
"logits/rejected": -36155372.76995305,
|
|
"logps/chosen": -504.7025643525741,
|
|
"logps/rejected": -390.96909233176837,
|
|
"loss": 0.5765,
|
|
"loss/base_kto": 3.1824538707733154,
|
|
"metrics/advantage_var": 801.2633056640625,
|
|
"regularization/mmd": 1.4298820495605469,
|
|
"regularization/rkhs_norm": 275.5071716308594,
|
|
"rewards/chosen": 0.5310748928980596,
|
|
"rewards/margins": 0.9131931776936386,
|
|
"rewards/rejected": -0.382118284795579,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 12.155492782592773,
|
|
"kl": 12.634875297546387,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -33998885.40639269,
|
|
"logits/rejected": -35240964.93097913,
|
|
"logps/chosen": -510.9780726788432,
|
|
"logps/rejected": -367.4885132423756,
|
|
"loss": 0.5634,
|
|
"loss/base_kto": 3.177760124206543,
|
|
"metrics/advantage_var": 548.15625,
|
|
"regularization/mmd": 1.3298225402832031,
|
|
"regularization/rkhs_norm": 256.22784423828125,
|
|
"rewards/chosen": 0.5586960791089944,
|
|
"rewards/margins": 0.9076369330055258,
|
|
"rewards/rejected": -0.3489408538965314,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 11.107390403747559,
|
|
"kl": 14.30541706085205,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -33680413.703816794,
|
|
"logits/rejected": -36054204.416,
|
|
"logps/chosen": -457.80753816793896,
|
|
"logps/rejected": -373.3871,
|
|
"loss": 0.5567,
|
|
"loss/base_kto": 3.231868028640747,
|
|
"metrics/advantage_var": 455.01080322265625,
|
|
"regularization/mmd": 1.221943736076355,
|
|
"regularization/rkhs_norm": 235.4419403076172,
|
|
"rewards/chosen": 0.4908312149630248,
|
|
"rewards/margins": 0.8198550919161498,
|
|
"rewards/rejected": -0.329023876953125,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 15.35888671875,
|
|
"kl": 15.176299095153809,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34026542.40483384,
|
|
"logits/rejected": -35535394.79611651,
|
|
"logps/chosen": -481.6341106495468,
|
|
"logps/rejected": -390.0205805016181,
|
|
"loss": 0.569,
|
|
"loss/base_kto": 3.25787615776062,
|
|
"metrics/advantage_var": 572.3829956054688,
|
|
"regularization/mmd": 1.2940419912338257,
|
|
"regularization/rkhs_norm": 249.33372497558594,
|
|
"rewards/chosen": 0.5241825530176076,
|
|
"rewards/margins": 0.7955340926047334,
|
|
"rewards/rejected": -0.2713515395871258,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 15.133277893066406,
|
|
"kl": 16.57997703552246,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34742376.8576,
|
|
"logits/rejected": -35383405.4351145,
|
|
"logps/chosen": -466.6202,
|
|
"logps/rejected": -386.9606393129771,
|
|
"loss": 0.5684,
|
|
"loss/base_kto": 3.2192013263702393,
|
|
"metrics/advantage_var": 629.2428588867188,
|
|
"regularization/mmd": 1.3281208276748657,
|
|
"regularization/rkhs_norm": 255.8999481201172,
|
|
"rewards/chosen": 0.472026220703125,
|
|
"rewards/margins": 0.7915639702483899,
|
|
"rewards/rejected": -0.3195377495452648,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 10.430379867553711,
|
|
"kl": 13.760481834411621,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -33074653.866666667,
|
|
"logits/rejected": -36026586.65864661,
|
|
"logps/chosen": -500.7354674796748,
|
|
"logps/rejected": -401.15211466165414,
|
|
"loss": 0.5577,
|
|
"loss/base_kto": 3.153414487838745,
|
|
"metrics/advantage_var": 536.83984375,
|
|
"regularization/mmd": 1.3082083463668823,
|
|
"regularization/rkhs_norm": 252.063232421875,
|
|
"rewards/chosen": 0.5060722537157012,
|
|
"rewards/margins": 0.8814182955169818,
|
|
"rewards/rejected": -0.37534604180128056,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 11.11693000793457,
|
|
"kl": 10.771425247192383,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33730726.303759396,
|
|
"logits/rejected": -34242689.87317073,
|
|
"logps/chosen": -491.69915413533835,
|
|
"logps/rejected": -374.57708333333335,
|
|
"loss": 0.5635,
|
|
"loss/base_kto": 3.1932923793792725,
|
|
"metrics/advantage_var": 531.4647827148438,
|
|
"regularization/mmd": 1.3149811029434204,
|
|
"regularization/rkhs_norm": 253.3682098388672,
|
|
"rewards/chosen": 0.489731454490719,
|
|
"rewards/margins": 0.8919199866299465,
|
|
"rewards/rejected": -0.4021885321392276,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 12.586292266845703,
|
|
"kl": 20.481595993041992,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35130701.80787402,
|
|
"logits/rejected": -36678428.97364341,
|
|
"logps/chosen": -492.09655511811025,
|
|
"logps/rejected": -384.52284399224806,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.1790807247161865,
|
|
"metrics/advantage_var": 545.5796508789062,
|
|
"regularization/mmd": 1.3135855197906494,
|
|
"regularization/rkhs_norm": 253.0993194580078,
|
|
"rewards/chosen": 0.6542080616387795,
|
|
"rewards/margins": 0.8926301656389006,
|
|
"rewards/rejected": -0.23842210400012112,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 13.15192699432373,
|
|
"kl": 15.327397346496582,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34864882.27871363,
|
|
"logits/rejected": -37420546.449760765,
|
|
"logps/chosen": -516.3475306278714,
|
|
"logps/rejected": -353.7198963317384,
|
|
"loss": 0.5722,
|
|
"loss/base_kto": 3.1635806560516357,
|
|
"metrics/advantage_var": 609.1104736328125,
|
|
"regularization/mmd": 1.4136465787887573,
|
|
"regularization/rkhs_norm": 272.37890625,
|
|
"rewards/chosen": 0.6116748944175919,
|
|
"rewards/margins": 0.9000959412084861,
|
|
"rewards/rejected": -0.28842104679089414,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 16.29294204711914,
|
|
"kl": 13.822619438171387,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34549783.085346214,
|
|
"logits/rejected": -36527761.28679818,
|
|
"logps/chosen": -473.2524154589372,
|
|
"logps/rejected": -366.9097591047041,
|
|
"loss": 0.5574,
|
|
"loss/base_kto": 3.207738161087036,
|
|
"metrics/advantage_var": 483.0428161621094,
|
|
"regularization/mmd": 1.2514506578445435,
|
|
"regularization/rkhs_norm": 241.1272735595703,
|
|
"rewards/chosen": 0.5131281608544686,
|
|
"rewards/margins": 0.8448688686918642,
|
|
"rewards/rejected": -0.3317407078373957,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 11.576709747314453,
|
|
"kl": 12.038655281066895,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34267869.70716511,
|
|
"logits/rejected": -34709391.64890282,
|
|
"logps/chosen": -483.4979556074766,
|
|
"logps/rejected": -383.8982415752351,
|
|
"loss": 0.5645,
|
|
"loss/base_kto": 3.2420220375061035,
|
|
"metrics/advantage_var": 526.3206176757812,
|
|
"regularization/mmd": 1.2743748426437378,
|
|
"regularization/rkhs_norm": 245.5442657470703,
|
|
"rewards/chosen": 0.4424971345800477,
|
|
"rewards/margins": 0.8194643703724409,
|
|
"rewards/rejected": -0.3769672357923932,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 11.38870620727539,
|
|
"kl": 13.090853691101074,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33939830.185970634,
|
|
"logits/rejected": -36001134.92053973,
|
|
"logps/chosen": -470.4476447797716,
|
|
"logps/rejected": -368.1530640929535,
|
|
"loss": 0.558,
|
|
"loss/base_kto": 3.1668918132781982,
|
|
"metrics/advantage_var": 513.3076171875,
|
|
"regularization/mmd": 1.297245740890503,
|
|
"regularization/rkhs_norm": 249.9510040283203,
|
|
"rewards/chosen": 0.517286988300367,
|
|
"rewards/margins": 0.893335452410597,
|
|
"rewards/rejected": -0.37604846411023,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 13.650389671325684,
|
|
"kl": 8.224946975708008,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33210067.37614679,
|
|
"logits/rejected": -35512692.958466455,
|
|
"logps/chosen": -473.4014239296636,
|
|
"logps/rejected": -364.13248801916933,
|
|
"loss": 0.5638,
|
|
"loss/base_kto": 3.248142957687378,
|
|
"metrics/advantage_var": 484.4810485839844,
|
|
"regularization/mmd": 1.2620840072631836,
|
|
"regularization/rkhs_norm": 243.17613220214844,
|
|
"rewards/chosen": 0.4389438454164277,
|
|
"rewards/margins": 0.8400392753457284,
|
|
"rewards/rejected": -0.40109542992930064,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 35.175785064697266,
|
|
"kl": 17.141820907592773,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35519068.86677367,
|
|
"logits/rejected": -35112560.99847793,
|
|
"logps/chosen": -500.48475120385234,
|
|
"logps/rejected": -396.3122621765601,
|
|
"loss": 0.5787,
|
|
"loss/base_kto": 3.1796255111694336,
|
|
"metrics/advantage_var": 821.1322631835938,
|
|
"regularization/mmd": 1.4498852491378784,
|
|
"regularization/rkhs_norm": 279.361328125,
|
|
"rewards/chosen": 0.5144086565290179,
|
|
"rewards/margins": 0.8857092014419745,
|
|
"rewards/rejected": -0.3713005449129566,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.471954345703125,
|
|
"kl": 6.620677471160889,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33879606.4,
|
|
"logits/rejected": -36046924.8,
|
|
"logps/chosen": -471.56845703125,
|
|
"logps/rejected": -379.3955810546875,
|
|
"loss": 0.5753,
|
|
"loss/base_kto": 3.2660164833068848,
|
|
"metrics/advantage_var": 629.7620239257812,
|
|
"regularization/mmd": 1.336260437965393,
|
|
"regularization/rkhs_norm": 257.4682922363281,
|
|
"rewards/chosen": 0.41168975830078125,
|
|
"rewards/margins": 0.8532877922058105,
|
|
"rewards/rejected": -0.4415980339050293,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 17.38668441772461,
|
|
"kl": 12.52214241027832,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -35436497.94302849,
|
|
"logits/rejected": -35837878.49918434,
|
|
"logps/chosen": -469.97985382308843,
|
|
"logps/rejected": -359.86421798531813,
|
|
"loss": 0.5749,
|
|
"loss/base_kto": 3.249713659286499,
|
|
"metrics/advantage_var": 892.2927856445312,
|
|
"regularization/mmd": 1.34954833984375,
|
|
"regularization/rkhs_norm": 260.028564453125,
|
|
"rewards/chosen": 0.516921745426115,
|
|
"rewards/margins": 0.7756842730661793,
|
|
"rewards/rejected": -0.25876252764006424,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 11.343464851379395,
|
|
"kl": 12.849252700805664,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34218797.99046105,
|
|
"logits/rejected": -35599304.94623656,
|
|
"logps/chosen": -496.7043918918919,
|
|
"logps/rejected": -386.3777361751152,
|
|
"loss": 0.5582,
|
|
"loss/base_kto": 3.1670498847961426,
|
|
"metrics/advantage_var": 505.1261901855469,
|
|
"regularization/mmd": 1.2983702421188354,
|
|
"regularization/rkhs_norm": 250.1676788330078,
|
|
"rewards/chosen": 0.5697533830360691,
|
|
"rewards/margins": 0.883675239659893,
|
|
"rewards/rejected": -0.31392185662382394,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 12.956347465515137,
|
|
"kl": 9.079574584960938,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35288816.42433697,
|
|
"logits/rejected": -35891449.99061033,
|
|
"logps/chosen": -480.0551872074883,
|
|
"logps/rejected": -364.0055751173709,
|
|
"loss": 0.5746,
|
|
"loss/base_kto": 3.2379226684570312,
|
|
"metrics/advantage_var": 586.6058959960938,
|
|
"regularization/mmd": 1.3591790199279785,
|
|
"regularization/rkhs_norm": 261.88421630859375,
|
|
"rewards/chosen": 0.43446138236154447,
|
|
"rewards/margins": 0.8490052864108402,
|
|
"rewards/rejected": -0.41454390404929575,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 9.659762382507324,
|
|
"kl": 8.767071723937988,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -33956107.41401274,
|
|
"logits/rejected": -35763363.33742331,
|
|
"logps/chosen": -466.2066580414013,
|
|
"logps/rejected": -391.6272526840491,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.2862496376037598,
|
|
"metrics/advantage_var": 492.1122131347656,
|
|
"regularization/mmd": 1.231597900390625,
|
|
"regularization/rkhs_norm": 237.30210876464844,
|
|
"rewards/chosen": 0.3718239365109972,
|
|
"rewards/margins": 0.7595649317659818,
|
|
"rewards/rejected": -0.3877409952549847,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 12.61690616607666,
|
|
"kl": 9.680636405944824,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34631756.68018721,
|
|
"logits/rejected": -37399765.13302034,
|
|
"logps/chosen": -488.1589313572543,
|
|
"logps/rejected": -391.39862089201876,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.236057996749878,
|
|
"metrics/advantage_var": 480.9098205566406,
|
|
"regularization/mmd": 1.2427895069122314,
|
|
"regularization/rkhs_norm": 239.45849609375,
|
|
"rewards/chosen": 0.43861253473576445,
|
|
"rewards/margins": 0.8102007506905644,
|
|
"rewards/rejected": -0.37158821595479996,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 13.531383514404297,
|
|
"kl": 9.788293838500977,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33087113.32713178,
|
|
"logits/rejected": -34771073.00787402,
|
|
"logps/chosen": -505.1265503875969,
|
|
"logps/rejected": -365.8290846456693,
|
|
"loss": 0.5659,
|
|
"loss/base_kto": 3.218250036239624,
|
|
"metrics/advantage_var": 558.4009399414062,
|
|
"regularization/mmd": 1.3091548681259155,
|
|
"regularization/rkhs_norm": 252.2456512451172,
|
|
"rewards/chosen": 0.48601793392684106,
|
|
"rewards/margins": 0.8748068330256353,
|
|
"rewards/rejected": -0.38878889909879427,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 10.86054515838623,
|
|
"kl": 11.343132972717285,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -32781315.36842105,
|
|
"logits/rejected": -34524117.333333336,
|
|
"logps/chosen": -491.22974917763156,
|
|
"logps/rejected": -369.2662760416667,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.237612247467041,
|
|
"metrics/advantage_var": 437.27813720703125,
|
|
"regularization/mmd": 1.2136201858520508,
|
|
"regularization/rkhs_norm": 233.8381805419922,
|
|
"rewards/chosen": 0.4579402522036904,
|
|
"rewards/margins": 0.795162646692797,
|
|
"rewards/rejected": -0.3372223944891067,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 8.429838180541992,
|
|
"kl": 4.938611030578613,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35576375.98732171,
|
|
"logits/rejected": -35768968.90262751,
|
|
"logps/chosen": -500.66333201267827,
|
|
"logps/rejected": -373.9417986862442,
|
|
"loss": 0.5488,
|
|
"loss/base_kto": 3.1686315536499023,
|
|
"metrics/advantage_var": 463.1637878417969,
|
|
"regularization/mmd": 1.2215324640274048,
|
|
"regularization/rkhs_norm": 235.36270141601562,
|
|
"rewards/chosen": 0.37699362970947653,
|
|
"rewards/margins": 0.9123040241380893,
|
|
"rewards/rejected": -0.5353103944286128,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 15.727373123168945,
|
|
"kl": 10.224392890930176,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34169435.11181103,
|
|
"logits/rejected": -35353080.85581395,
|
|
"logps/chosen": -483.05826771653545,
|
|
"logps/rejected": -390.73168604651164,
|
|
"loss": 0.5299,
|
|
"loss/base_kto": 3.1048107147216797,
|
|
"metrics/advantage_var": 380.64837646484375,
|
|
"regularization/mmd": 1.1341379880905151,
|
|
"regularization/rkhs_norm": 218.52371215820312,
|
|
"rewards/chosen": 0.4996672863096703,
|
|
"rewards/margins": 0.9463607895194571,
|
|
"rewards/rejected": -0.44669350320978685,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 13.826533317565918,
|
|
"kl": 9.03232479095459,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34003890.42424242,
|
|
"logits/rejected": -34565113.393548384,
|
|
"logps/chosen": -479.08357007575756,
|
|
"logps/rejected": -382.40057963709677,
|
|
"loss": 0.5293,
|
|
"loss/base_kto": 3.0912117958068848,
|
|
"metrics/advantage_var": 367.74786376953125,
|
|
"regularization/mmd": 1.143200159072876,
|
|
"regularization/rkhs_norm": 220.269775390625,
|
|
"rewards/chosen": 0.5297567656545928,
|
|
"rewards/margins": 0.9286658114585242,
|
|
"rewards/rejected": -0.39890904580393144,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 10.95798110961914,
|
|
"kl": 9.553214073181152,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34640685.2733119,
|
|
"logits/rejected": -36745863.392097265,
|
|
"logps/chosen": -479.6584103697749,
|
|
"logps/rejected": -407.0407959726444,
|
|
"loss": 0.5306,
|
|
"loss/base_kto": 3.100395917892456,
|
|
"metrics/advantage_var": 376.327392578125,
|
|
"regularization/mmd": 1.144347071647644,
|
|
"regularization/rkhs_norm": 220.4907684326172,
|
|
"rewards/chosen": 0.5258300388740956,
|
|
"rewards/margins": 0.9229663212876595,
|
|
"rewards/rejected": -0.39713628241356386,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 10.353028297424316,
|
|
"kl": 10.044342041015625,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34038157.42097027,
|
|
"logits/rejected": -35445748.8174727,
|
|
"logps/chosen": -455.44600938967136,
|
|
"logps/rejected": -377.6342628705148,
|
|
"loss": 0.5329,
|
|
"loss/base_kto": 3.1051576137542725,
|
|
"metrics/advantage_var": 379.6630554199219,
|
|
"regularization/mmd": 1.1578160524368286,
|
|
"regularization/rkhs_norm": 223.0859375,
|
|
"rewards/chosen": 0.5091337821852993,
|
|
"rewards/margins": 0.9376146238115424,
|
|
"rewards/rejected": -0.42848084162624317,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 9.661334037780762,
|
|
"kl": 9.330436706542969,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35977493.590361446,
|
|
"logits/rejected": -36188432.62337662,
|
|
"logps/chosen": -475.7725903614458,
|
|
"logps/rejected": -377.3262987012987,
|
|
"loss": 0.5294,
|
|
"loss/base_kto": 3.0793023109436035,
|
|
"metrics/advantage_var": 376.308837890625,
|
|
"regularization/mmd": 1.1555709838867188,
|
|
"regularization/rkhs_norm": 222.6533660888672,
|
|
"rewards/chosen": 0.5504086965537933,
|
|
"rewards/margins": 0.9590159565270964,
|
|
"rewards/rejected": -0.40860725997330305,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 12.885091781616211,
|
|
"kl": 13.551025390625,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34820204.29237947,
|
|
"logits/rejected": -36722845.538461536,
|
|
"logps/chosen": -498.05234253499225,
|
|
"logps/rejected": -364.3190737833595,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.0458972454071045,
|
|
"metrics/advantage_var": 389.7593688964844,
|
|
"regularization/mmd": 1.1493853330612183,
|
|
"regularization/rkhs_norm": 221.4615478515625,
|
|
"rewards/chosen": 0.6288376685063909,
|
|
"rewards/margins": 0.9763314136005824,
|
|
"rewards/rejected": -0.3474937450941915,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 13.358975410461426,
|
|
"kl": 12.604578018188477,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34915930.16352201,
|
|
"logits/rejected": -36279900.22360248,
|
|
"logps/chosen": -478.5619103773585,
|
|
"logps/rejected": -371.0585209627329,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.1058311462402344,
|
|
"metrics/advantage_var": 371.24053955078125,
|
|
"regularization/mmd": 1.1212542057037354,
|
|
"regularization/rkhs_norm": 216.04129028320312,
|
|
"rewards/chosen": 0.5444417509642787,
|
|
"rewards/margins": 0.9048593717001581,
|
|
"rewards/rejected": -0.3604176207358793,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 12.47192096710205,
|
|
"kl": 13.442543029785156,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34275425.52380952,
|
|
"logits/rejected": -34517924.627692305,
|
|
"logps/chosen": -466.6615079365079,
|
|
"logps/rejected": -383.03625,
|
|
"loss": 0.5273,
|
|
"loss/base_kto": 3.1435506343841553,
|
|
"metrics/advantage_var": 328.599853515625,
|
|
"regularization/mmd": 1.075141191482544,
|
|
"regularization/rkhs_norm": 207.15628051757812,
|
|
"rewards/chosen": 0.49446861630394345,
|
|
"rewards/margins": 0.857264636811756,
|
|
"rewards/rejected": -0.3627960205078125,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 12.349054336547852,
|
|
"kl": 9.524312019348145,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35792114.87179487,
|
|
"logits/rejected": -37086373.46341463,
|
|
"logps/chosen": -477.56275040064105,
|
|
"logps/rejected": -368.961318597561,
|
|
"loss": 0.5267,
|
|
"loss/base_kto": 3.122302293777466,
|
|
"metrics/advantage_var": 341.88995361328125,
|
|
"regularization/mmd": 1.0910618305206299,
|
|
"regularization/rkhs_norm": 210.2238311767578,
|
|
"rewards/chosen": 0.4709402720133464,
|
|
"rewards/margins": 0.9085386167696821,
|
|
"rewards/rejected": -0.43759834475633574,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 14.046501159667969,
|
|
"kl": 9.571139335632324,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34408404.688821755,
|
|
"logits/rejected": -35771739.96116505,
|
|
"logps/chosen": -487.6857533987915,
|
|
"logps/rejected": -368.93782868122975,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.106714963912964,
|
|
"metrics/advantage_var": 336.606201171875,
|
|
"regularization/mmd": 1.0885037183761597,
|
|
"regularization/rkhs_norm": 209.73095703125,
|
|
"rewards/chosen": 0.5020798974166824,
|
|
"rewards/margins": 0.9009448904164042,
|
|
"rewards/rejected": -0.3988649929997219,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 8.706116676330566,
|
|
"kl": 15.946802139282227,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34518271.19242902,
|
|
"logits/rejected": -35398031.45510836,
|
|
"logps/chosen": -485.34611593059935,
|
|
"logps/rejected": -389.8077109133127,
|
|
"loss": 0.5297,
|
|
"loss/base_kto": 3.0599441528320312,
|
|
"metrics/advantage_var": 391.05816650390625,
|
|
"regularization/mmd": 1.17766273021698,
|
|
"regularization/rkhs_norm": 226.9099578857422,
|
|
"rewards/chosen": 0.6544270801243346,
|
|
"rewards/margins": 0.959031412637819,
|
|
"rewards/rejected": -0.3046043325134844,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 13.162550926208496,
|
|
"kl": 10.673392295837402,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34212820.02453988,
|
|
"logits/rejected": -35669468.12738854,
|
|
"logps/chosen": -465.9872028374233,
|
|
"logps/rejected": -371.60225915605093,
|
|
"loss": 0.5275,
|
|
"loss/base_kto": 3.1158010959625244,
|
|
"metrics/advantage_var": 348.5242614746094,
|
|
"regularization/mmd": 1.1044648885726929,
|
|
"regularization/rkhs_norm": 212.8063507080078,
|
|
"rewards/chosen": 0.5190921151564897,
|
|
"rewards/margins": 0.9168897665081268,
|
|
"rewards/rejected": -0.39779765135163714,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 10.201983451843262,
|
|
"kl": 13.881949424743652,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35078533.94902549,
|
|
"logits/rejected": -36514550.39477977,
|
|
"logps/chosen": -525.0691529235382,
|
|
"logps/rejected": -358.8348796900489,
|
|
"loss": 0.5295,
|
|
"loss/base_kto": 3.0585391521453857,
|
|
"metrics/advantage_var": 389.27783203125,
|
|
"regularization/mmd": 1.1772745847702026,
|
|
"regularization/rkhs_norm": 226.83518981933594,
|
|
"rewards/chosen": 0.6114501953125,
|
|
"rewards/margins": 0.9704890538973288,
|
|
"rewards/rejected": -0.3590388585848287,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 8.7860746383667,
|
|
"kl": 15.788287162780762,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33913246.059097975,
|
|
"logits/rejected": -35752505.06750392,
|
|
"logps/chosen": -490.89900855365477,
|
|
"logps/rejected": -366.88947213500785,
|
|
"loss": 0.5299,
|
|
"loss/base_kto": 3.11326003074646,
|
|
"metrics/advantage_var": 357.09552001953125,
|
|
"regularization/mmd": 1.1259396076202393,
|
|
"regularization/rkhs_norm": 216.9440460205078,
|
|
"rewards/chosen": 0.6084082145156979,
|
|
"rewards/margins": 0.9001660346810236,
|
|
"rewards/rejected": -0.29175782016532575,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 31.27370262145996,
|
|
"kl": 15.632532119750977,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34557501.244019136,
|
|
"logits/rejected": -35704850.81776416,
|
|
"logps/chosen": -450.49601275917064,
|
|
"logps/rejected": -386.5971238514548,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.090442657470703,
|
|
"metrics/advantage_var": 366.620361328125,
|
|
"regularization/mmd": 1.1398146152496338,
|
|
"regularization/rkhs_norm": 219.61746215820312,
|
|
"rewards/chosen": 0.5583569589033842,
|
|
"rewards/margins": 0.9100697334400778,
|
|
"rewards/rejected": -0.35171277453669364,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 12.42688274383545,
|
|
"kl": 13.420425415039062,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33294260.381538462,
|
|
"logits/rejected": -35762162.9968254,
|
|
"logps/chosen": -475.5403846153846,
|
|
"logps/rejected": -371.23055555555555,
|
|
"loss": 0.5364,
|
|
"loss/base_kto": 3.106178045272827,
|
|
"metrics/advantage_var": 407.9663391113281,
|
|
"regularization/mmd": 1.1849819421768188,
|
|
"regularization/rkhs_norm": 228.32022094726562,
|
|
"rewards/chosen": 0.5992143366887019,
|
|
"rewards/margins": 0.9340711753388755,
|
|
"rewards/rejected": -0.3348568386501736,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.732342720031738,
|
|
"kl": 12.329475402832031,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -33890418.526315786,
|
|
"logits/rejected": -36418870.85714286,
|
|
"logps/chosen": -475.4996402138158,
|
|
"logps/rejected": -377.1099330357143,
|
|
"loss": 0.5301,
|
|
"loss/base_kto": 3.1279115676879883,
|
|
"metrics/advantage_var": 363.8894958496094,
|
|
"regularization/mmd": 1.1131722927093506,
|
|
"regularization/rkhs_norm": 214.48403930664062,
|
|
"rewards/chosen": 0.5284870549252159,
|
|
"rewards/margins": 0.8872456180123159,
|
|
"rewards/rejected": -0.3587585630871001,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 13.289918899536133,
|
|
"kl": 9.346504211425781,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34114033.82153846,
|
|
"logits/rejected": -36181222.8063492,
|
|
"logps/chosen": -478.6663461538462,
|
|
"logps/rejected": -362.11063988095236,
|
|
"loss": 0.5295,
|
|
"loss/base_kto": 3.1132397651672363,
|
|
"metrics/advantage_var": 358.97418212890625,
|
|
"regularization/mmd": 1.1230419874191284,
|
|
"regularization/rkhs_norm": 216.3857421875,
|
|
"rewards/chosen": 0.5090151742788461,
|
|
"rewards/margins": 0.9224266326121795,
|
|
"rewards/rejected": -0.4134114583333333,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 16.736160278320312,
|
|
"kl": 10.813815116882324,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -33832985.801574804,
|
|
"logits/rejected": -35365714.95193798,
|
|
"logps/chosen": -509.0462598425197,
|
|
"logps/rejected": -384.3090843023256,
|
|
"loss": 0.532,
|
|
"loss/base_kto": 3.09063982963562,
|
|
"metrics/advantage_var": 386.1016845703125,
|
|
"regularization/mmd": 1.1649986505508423,
|
|
"regularization/rkhs_norm": 224.46987915039062,
|
|
"rewards/chosen": 0.5305650125338337,
|
|
"rewards/margins": 0.9422878315442504,
|
|
"rewards/rejected": -0.4117228190104167,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.82371997833252,
|
|
"kl": 12.002352714538574,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34679065.8048,
|
|
"logits/rejected": -36309939.39541985,
|
|
"logps/chosen": -460.6847,
|
|
"logps/rejected": -379.85696564885495,
|
|
"loss": 0.5252,
|
|
"loss/base_kto": 3.0716331005096436,
|
|
"metrics/advantage_var": 372.6279602050781,
|
|
"regularization/mmd": 1.1301687955856323,
|
|
"regularization/rkhs_norm": 217.75889587402344,
|
|
"rewards/chosen": 0.5291451171875,
|
|
"rewards/margins": 0.939521233525167,
|
|
"rewards/rejected": -0.41037611633766696,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 12.417250633239746,
|
|
"kl": 9.33301830291748,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35359239.87692308,
|
|
"logits/rejected": -35752865.72698413,
|
|
"logps/chosen": -484.42115384615386,
|
|
"logps/rejected": -395.8510912698413,
|
|
"loss": 0.5325,
|
|
"loss/base_kto": 3.1038146018981934,
|
|
"metrics/advantage_var": 398.2544860839844,
|
|
"regularization/mmd": 1.156279444694519,
|
|
"regularization/rkhs_norm": 222.7898406982422,
|
|
"rewards/chosen": 0.5300423490084135,
|
|
"rewards/margins": 0.9482267565604969,
|
|
"rewards/rejected": -0.41818440755208336,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 11.20324420928955,
|
|
"kl": 13.021185874938965,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34471727.84262295,
|
|
"logits/rejected": -35850325.5880597,
|
|
"logps/chosen": -475.0947745901639,
|
|
"logps/rejected": -402.4258395522388,
|
|
"loss": 0.5245,
|
|
"loss/base_kto": 3.1126742362976074,
|
|
"metrics/advantage_var": 339.88214111328125,
|
|
"regularization/mmd": 1.0831588506698608,
|
|
"regularization/rkhs_norm": 208.7011260986328,
|
|
"rewards/chosen": 0.5231860551677766,
|
|
"rewards/margins": 0.9091018284740966,
|
|
"rewards/rejected": -0.38591577330632,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 13.183117866516113,
|
|
"kl": 8.468612670898438,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34805089.27186009,
|
|
"logits/rejected": -35219817.78187404,
|
|
"logps/chosen": -507.5408386327504,
|
|
"logps/rejected": -360.90430587557603,
|
|
"loss": 0.5339,
|
|
"loss/base_kto": 3.110563039779663,
|
|
"metrics/advantage_var": 389.3725280761719,
|
|
"regularization/mmd": 1.1603578329086304,
|
|
"regularization/rkhs_norm": 223.57568359375,
|
|
"rewards/chosen": 0.5026676923936805,
|
|
"rewards/margins": 0.9296209861544812,
|
|
"rewards/rejected": -0.4269532937608007,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 10.095125198364258,
|
|
"kl": 9.794113159179688,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34838252.8,
|
|
"logits/rejected": -35133248.0,
|
|
"logps/chosen": -499.425,
|
|
"logps/rejected": -408.65634765625,
|
|
"loss": 0.5297,
|
|
"loss/base_kto": 3.0677144527435303,
|
|
"metrics/advantage_var": 388.46734619140625,
|
|
"regularization/mmd": 1.1697847843170166,
|
|
"regularization/rkhs_norm": 225.3920440673828,
|
|
"rewards/chosen": 0.4837846279144287,
|
|
"rewards/margins": 0.9642529010772705,
|
|
"rewards/rejected": -0.4804682731628418,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 10.045802116394043,
|
|
"kl": 10.972128868103027,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34800860.8,
|
|
"logits/rejected": -35979273.6,
|
|
"logps/chosen": -498.69697265625,
|
|
"logps/rejected": -378.28642578125,
|
|
"loss": 0.5287,
|
|
"loss/base_kto": 3.090186357498169,
|
|
"metrics/advantage_var": 368.100341796875,
|
|
"regularization/mmd": 1.1397637128829956,
|
|
"regularization/rkhs_norm": 219.607666015625,
|
|
"rewards/chosen": 0.5003337860107422,
|
|
"rewards/margins": 0.9343658924102783,
|
|
"rewards/rejected": -0.43403210639953616,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 11.280572891235352,
|
|
"kl": 8.247645378112793,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -33784674.11453745,
|
|
"logits/rejected": -36023906.29716194,
|
|
"logps/chosen": -503.7002569750367,
|
|
"logps/rejected": -352.47876669449084,
|
|
"loss": 0.5397,
|
|
"loss/base_kto": 3.07843017578125,
|
|
"metrics/advantage_var": 763.4011840820312,
|
|
"regularization/mmd": 1.2387841939926147,
|
|
"regularization/rkhs_norm": 238.68675231933594,
|
|
"rewards/chosen": 0.557862269195691,
|
|
"rewards/margins": 0.9298415893842866,
|
|
"rewards/rejected": -0.3719793201885956,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 8.601116180419922,
|
|
"kl": 11.35707950592041,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -32804689.56466877,
|
|
"logits/rejected": -35856808.81733746,
|
|
"logps/chosen": -475.0592468454259,
|
|
"logps/rejected": -353.30098684210526,
|
|
"loss": 0.5302,
|
|
"loss/base_kto": 3.1280622482299805,
|
|
"metrics/advantage_var": 365.3536682128906,
|
|
"regularization/mmd": 1.113821268081665,
|
|
"regularization/rkhs_norm": 214.6090850830078,
|
|
"rewards/chosen": 0.501492665769174,
|
|
"rewards/margins": 0.8903800770697858,
|
|
"rewards/rejected": -0.38888741130061194,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 12.126202583312988,
|
|
"kl": 13.453925132751465,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34088487.384615384,
|
|
"logits/rejected": -36759580.665629864,
|
|
"logps/chosen": -492.50539638932497,
|
|
"logps/rejected": -376.3147113141524,
|
|
"loss": 0.5308,
|
|
"loss/base_kto": 3.071422815322876,
|
|
"metrics/advantage_var": 398.5673828125,
|
|
"regularization/mmd": 1.1750009059906006,
|
|
"regularization/rkhs_norm": 226.39707946777344,
|
|
"rewards/chosen": 0.5706526009216788,
|
|
"rewards/margins": 0.9671484140808205,
|
|
"rewards/rejected": -0.39649581315914173,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 10.9151029586792,
|
|
"kl": 9.662836074829102,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35570628.13538461,
|
|
"logits/rejected": -37069053.56190476,
|
|
"logps/chosen": -485.08846153846156,
|
|
"logps/rejected": -382.0393353174603,
|
|
"loss": 0.5296,
|
|
"loss/base_kto": 3.0812783241271973,
|
|
"metrics/advantage_var": 394.0767517089844,
|
|
"regularization/mmd": 1.1554718017578125,
|
|
"regularization/rkhs_norm": 222.63426208496094,
|
|
"rewards/chosen": 0.5557008713942307,
|
|
"rewards/margins": 0.972709480257698,
|
|
"rewards/rejected": -0.41700860886346724,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 10.44419002532959,
|
|
"kl": 11.893241882324219,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34611472.20253164,
|
|
"logits/rejected": -34804666.469135806,
|
|
"logps/chosen": -483.117929193038,
|
|
"logps/rejected": -354.1086516203704,
|
|
"loss": 0.5247,
|
|
"loss/base_kto": 3.085773229598999,
|
|
"metrics/advantage_var": 369.8644104003906,
|
|
"regularization/mmd": 1.1118484735488892,
|
|
"regularization/rkhs_norm": 214.22900390625,
|
|
"rewards/chosen": 0.5326901206487342,
|
|
"rewards/margins": 0.9465245865233002,
|
|
"rewards/rejected": -0.41383446587456596,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 9.923108100891113,
|
|
"kl": 11.843527793884277,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35572256.820512824,
|
|
"logits/rejected": -34997360.3902439,
|
|
"logps/chosen": -460.6005608974359,
|
|
"logps/rejected": -404.8897198932927,
|
|
"loss": 0.5256,
|
|
"loss/base_kto": 3.0892350673675537,
|
|
"metrics/advantage_var": 350.0757751464844,
|
|
"regularization/mmd": 1.1153289079666138,
|
|
"regularization/rkhs_norm": 214.8996124267578,
|
|
"rewards/chosen": 0.48085985428247696,
|
|
"rewards/margins": 0.9150912634949151,
|
|
"rewards/rejected": -0.4342314092124381,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 14.040332794189453,
|
|
"kl": 11.11498737335205,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -33898979.95618153,
|
|
"logits/rejected": -34968705.39781591,
|
|
"logps/chosen": -464.94874804381845,
|
|
"logps/rejected": -375.38672484399376,
|
|
"loss": 0.5275,
|
|
"loss/base_kto": 3.1421051025390625,
|
|
"metrics/advantage_var": 339.7447204589844,
|
|
"regularization/mmd": 1.078227162361145,
|
|
"regularization/rkhs_norm": 207.75088500976562,
|
|
"rewards/chosen": 0.5223304617199727,
|
|
"rewards/margins": 0.8760799913398294,
|
|
"rewards/rejected": -0.3537495296198567,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 9.759392738342285,
|
|
"kl": 10.804986953735352,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35574749.01863354,
|
|
"logits/rejected": -36234394.56603774,
|
|
"logps/chosen": -496.5819099378882,
|
|
"logps/rejected": -370.5633352987421,
|
|
"loss": 0.5302,
|
|
"loss/base_kto": 3.097810745239258,
|
|
"metrics/advantage_var": 378.4516296386719,
|
|
"regularization/mmd": 1.1435234546661377,
|
|
"regularization/rkhs_norm": 220.3320770263672,
|
|
"rewards/chosen": 0.5362494332449776,
|
|
"rewards/margins": 0.9271788892720265,
|
|
"rewards/rejected": -0.3909294560270489,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 9.869982719421387,
|
|
"kl": 11.468153953552246,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34745472.80503145,
|
|
"logits/rejected": -36531956.86956522,
|
|
"logps/chosen": -470.8713639937107,
|
|
"logps/rejected": -372.5007278726708,
|
|
"loss": 0.5236,
|
|
"loss/base_kto": 3.0981380939483643,
|
|
"metrics/advantage_var": 346.8184509277344,
|
|
"regularization/mmd": 1.0902889966964722,
|
|
"regularization/rkhs_norm": 210.074951171875,
|
|
"rewards/chosen": 0.5356377175768966,
|
|
"rewards/margins": 0.915488519791146,
|
|
"rewards/rejected": -0.37985080221424933,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 14.449504852294922,
|
|
"kl": 11.762713432312012,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35228356.682926826,
|
|
"logits/rejected": -35463424.0,
|
|
"logps/chosen": -469.6153296493902,
|
|
"logps/rejected": -390.17052283653845,
|
|
"loss": 0.5292,
|
|
"loss/base_kto": 3.1139495372772217,
|
|
"metrics/advantage_var": 372.5120544433594,
|
|
"regularization/mmd": 1.1196266412734985,
|
|
"regularization/rkhs_norm": 215.72769165039062,
|
|
"rewards/chosen": 0.5367694017363758,
|
|
"rewards/margins": 0.9177948138801808,
|
|
"rewards/rejected": -0.3810254121438051,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 11.837974548339844,
|
|
"kl": 11.047762870788574,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34728321.01105845,
|
|
"logits/rejected": -36686320.96445131,
|
|
"logps/chosen": -478.1500789889416,
|
|
"logps/rejected": -363.3257341576507,
|
|
"loss": 0.5278,
|
|
"loss/base_kto": 3.11044979095459,
|
|
"metrics/advantage_var": 355.1884765625,
|
|
"regularization/mmd": 1.1120823621749878,
|
|
"regularization/rkhs_norm": 214.2740478515625,
|
|
"rewards/chosen": 0.5352003148758393,
|
|
"rewards/margins": 0.9138535802849994,
|
|
"rewards/rejected": -0.37865326540916006,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 12.430468559265137,
|
|
"kl": 12.598543167114258,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33505824.851330202,
|
|
"logits/rejected": -35347122.121684864,
|
|
"logps/chosen": -475.14074726134584,
|
|
"logps/rejected": -352.0942618954758,
|
|
"loss": 0.5259,
|
|
"loss/base_kto": 3.089165210723877,
|
|
"metrics/advantage_var": 367.65313720703125,
|
|
"regularization/mmd": 1.1177597045898438,
|
|
"regularization/rkhs_norm": 215.3679656982422,
|
|
"rewards/chosen": 0.5344488534942292,
|
|
"rewards/margins": 0.9333023637869846,
|
|
"rewards/rejected": -0.39885351029275545,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_loss": 0.5590571298912194,
|
|
"train_runtime": 11070.9945,
|
|
"train_samples_per_second": 13.388,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|