1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 9.96064281463623,
|
|
"kl": 57.65949249267578,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37995714.74846626,
|
|
"logits/rejected": -38779241.98726115,
|
|
"logps/chosen": -450.8286042944785,
|
|
"logps/rejected": -374.59121218152865,
|
|
"loss": 0.4564,
|
|
"loss/base_kto": 3.6511714458465576,
|
|
"metrics/advantage_var": 538.0438842773438,
|
|
"rewards/chosen": 0.9948832506051093,
|
|
"rewards/margins": 0.29010821180696533,
|
|
"rewards/rejected": 0.704775038798144,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 10.330055236816406,
|
|
"kl": 16.56190299987793,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37086065.86750789,
|
|
"logits/rejected": -37968436.30959752,
|
|
"logps/chosen": -475.0192724763407,
|
|
"logps/rejected": -362.4485777863777,
|
|
"loss": 0.4488,
|
|
"loss/base_kto": 3.590278387069702,
|
|
"metrics/advantage_var": 806.6080932617188,
|
|
"rewards/chosen": 0.528348327059099,
|
|
"rewards/margins": 0.4430706338686572,
|
|
"rewards/rejected": 0.08527769319044178,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 8.40238094329834,
|
|
"kl": 10.559950828552246,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36252843.67058823,
|
|
"logits/rejected": -37554100.906666666,
|
|
"logps/chosen": -460.3076746323529,
|
|
"logps/rejected": -384.36171875,
|
|
"loss": 0.4338,
|
|
"loss/base_kto": 3.47013783454895,
|
|
"metrics/advantage_var": 966.3760986328125,
|
|
"rewards/chosen": 0.5126130047966452,
|
|
"rewards/margins": 0.6342480349073223,
|
|
"rewards/rejected": -0.12163503011067708,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 9.587729454040527,
|
|
"kl": 28.615070343017578,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37204144.38277512,
|
|
"logits/rejected": -38282552.061255746,
|
|
"logps/chosen": -479.75458532695376,
|
|
"logps/rejected": -382.3811255742726,
|
|
"loss": 0.441,
|
|
"loss/base_kto": 3.5277810096740723,
|
|
"metrics/advantage_var": 1355.2171630859375,
|
|
"rewards/chosen": 0.8166142757239334,
|
|
"rewards/margins": 0.5394598468709233,
|
|
"rewards/rejected": 0.27715442885301017,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 10.16214656829834,
|
|
"kl": 38.51455307006836,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -39859357.77643505,
|
|
"logits/rejected": -39609748.297734626,
|
|
"logps/chosen": -471.56891993957703,
|
|
"logps/rejected": -368.18031957928804,
|
|
"loss": 0.4119,
|
|
"loss/base_kto": 3.294884204864502,
|
|
"metrics/advantage_var": 2121.011474609375,
|
|
"rewards/chosen": 1.214173930649311,
|
|
"rewards/margins": 0.8701046318464691,
|
|
"rewards/rejected": 0.3440692988028418,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 9.195523262023926,
|
|
"kl": 18.842559814453125,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -39613262.46439628,
|
|
"logits/rejected": -38695997.37539432,
|
|
"logps/chosen": -462.6353037925697,
|
|
"logps/rejected": -347.01944499211356,
|
|
"loss": 0.4002,
|
|
"loss/base_kto": 3.2016677856445312,
|
|
"metrics/advantage_var": 3064.429443359375,
|
|
"rewards/chosen": 0.9390506331027477,
|
|
"rewards/margins": 1.1935873791549092,
|
|
"rewards/rejected": -0.25453674605216137,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 10.514404296875,
|
|
"kl": 8.773940086364746,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -40551948.17115689,
|
|
"logits/rejected": -40892785.20801233,
|
|
"logps/chosen": -469.73514263074486,
|
|
"logps/rejected": -388.4011941448382,
|
|
"loss": 0.3942,
|
|
"loss/base_kto": 3.1532838344573975,
|
|
"metrics/advantage_var": 4229.67578125,
|
|
"rewards/chosen": 0.5272595969319532,
|
|
"rewards/margins": 1.4671285885897922,
|
|
"rewards/rejected": -0.939868991657839,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 8.357895851135254,
|
|
"kl": 15.100295066833496,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -40738180.8607595,
|
|
"logits/rejected": -41197637.530864194,
|
|
"logps/chosen": -483.00178006329116,
|
|
"logps/rejected": -376.13421103395063,
|
|
"loss": 0.3844,
|
|
"loss/base_kto": 3.0754406452178955,
|
|
"metrics/advantage_var": 5060.587890625,
|
|
"rewards/chosen": 0.8644038333168512,
|
|
"rewards/margins": 1.6502909039609002,
|
|
"rewards/rejected": -0.785887070644049,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 9.702012062072754,
|
|
"kl": 6.5621843338012695,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -40952274.63291139,
|
|
"logits/rejected": -42449069.82716049,
|
|
"logps/chosen": -518.0392602848101,
|
|
"logps/rejected": -407.48533950617286,
|
|
"loss": 0.3861,
|
|
"loss/base_kto": 3.088801622390747,
|
|
"metrics/advantage_var": 4415.6943359375,
|
|
"rewards/chosen": 0.32813692696486846,
|
|
"rewards/margins": 1.63254242193738,
|
|
"rewards/rejected": -1.3044054949725117,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 7.299619674682617,
|
|
"kl": 16.3758544921875,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -42427388.73885351,
|
|
"logits/rejected": -42330005.202453986,
|
|
"logps/chosen": -513.141421178344,
|
|
"logps/rejected": -372.5459164110429,
|
|
"loss": 0.3698,
|
|
"loss/base_kto": 2.9581849575042725,
|
|
"metrics/advantage_var": 5061.00439453125,
|
|
"rewards/chosen": 0.7296892882912023,
|
|
"rewards/margins": 1.8715177667708802,
|
|
"rewards/rejected": -1.141828478479678,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 8.850095748901367,
|
|
"kl": 12.437146186828613,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -44007936.0,
|
|
"logits/rejected": -43484697.28395062,
|
|
"logps/chosen": -464.53258504746833,
|
|
"logps/rejected": -422.64699074074076,
|
|
"loss": 0.3858,
|
|
"loss/base_kto": 3.086326837539673,
|
|
"metrics/advantage_var": 5249.66796875,
|
|
"rewards/chosen": 0.7787781244591822,
|
|
"rewards/margins": 1.7817974808924384,
|
|
"rewards/rejected": -1.0030193564332561,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 9.223854064941406,
|
|
"kl": 34.16802978515625,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -41462785.627980925,
|
|
"logits/rejected": -39973147.13364055,
|
|
"logps/chosen": -478.3248211446741,
|
|
"logps/rejected": -391.4754704301075,
|
|
"loss": 0.3687,
|
|
"loss/base_kto": 2.9497230052948,
|
|
"metrics/advantage_var": 5468.2666015625,
|
|
"rewards/chosen": 1.6289144009588632,
|
|
"rewards/margins": 1.9584869936047926,
|
|
"rewards/rejected": -0.32957259264592936,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 7.754208564758301,
|
|
"kl": 18.512327194213867,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -42944346.443113774,
|
|
"logits/rejected": -42855497.62091503,
|
|
"logps/chosen": -464.94938248502996,
|
|
"logps/rejected": -359.9943831699346,
|
|
"loss": 0.3545,
|
|
"loss/base_kto": 2.8356099128723145,
|
|
"metrics/advantage_var": 5849.23486328125,
|
|
"rewards/chosen": 1.4158094029226702,
|
|
"rewards/margins": 2.1239714488689785,
|
|
"rewards/rejected": -0.7081620459463082,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 8.134888648986816,
|
|
"kl": 19.836334228515625,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -42694010.56969697,
|
|
"logits/rejected": -42238341.78064516,
|
|
"logps/chosen": -475.8674715909091,
|
|
"logps/rejected": -368.44627016129033,
|
|
"loss": 0.3552,
|
|
"loss/base_kto": 2.841984987258911,
|
|
"metrics/advantage_var": 5558.55859375,
|
|
"rewards/chosen": 1.2543638287168561,
|
|
"rewards/margins": 2.0791629541421335,
|
|
"rewards/rejected": -0.8247991254252772,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 7.4902472496032715,
|
|
"kl": 20.3782958984375,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -42808046.158833064,
|
|
"logits/rejected": -42098033.13423831,
|
|
"logps/chosen": -483.73936385737437,
|
|
"logps/rejected": -379.45880467571646,
|
|
"loss": 0.3512,
|
|
"loss/base_kto": 2.809675693511963,
|
|
"metrics/advantage_var": 6742.35791015625,
|
|
"rewards/chosen": 1.2607057840356564,
|
|
"rewards/margins": 2.351388560301211,
|
|
"rewards/rejected": -1.0906827762655542,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 11.804646492004395,
|
|
"kl": 15.350687026977539,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -43782101.4016,
|
|
"logits/rejected": -43546999.20610687,
|
|
"logps/chosen": -461.6126,
|
|
"logps/rejected": -410.2317748091603,
|
|
"loss": 0.3685,
|
|
"loss/base_kto": 2.9478862285614014,
|
|
"metrics/advantage_var": 7228.64404296875,
|
|
"rewards/chosen": 0.73833515625,
|
|
"rewards/margins": 2.2523628056416984,
|
|
"rewards/rejected": -1.5140276493916984,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 7.086822986602783,
|
|
"kl": 26.88038444519043,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -44641121.61715161,
|
|
"logits/rejected": -44243701.79266348,
|
|
"logps/chosen": -453.5135432618683,
|
|
"logps/rejected": -369.793610446571,
|
|
"loss": 0.3503,
|
|
"loss/base_kto": 2.802610397338867,
|
|
"metrics/advantage_var": 6313.2421875,
|
|
"rewards/chosen": 1.4890733049985643,
|
|
"rewards/margins": 2.2028278646265047,
|
|
"rewards/rejected": -0.7137545596279405,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 10.928313255310059,
|
|
"kl": 26.3825740814209,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -44585356.38709678,
|
|
"logits/rejected": -44008938.51230101,
|
|
"logps/chosen": -493.0664261460102,
|
|
"logps/rejected": -377.9354875180897,
|
|
"loss": 0.3429,
|
|
"loss/base_kto": 2.743035316467285,
|
|
"metrics/advantage_var": 7014.70947265625,
|
|
"rewards/chosen": 1.4837874459491722,
|
|
"rewards/margins": 2.5707175842790386,
|
|
"rewards/rejected": -1.0869301383298662,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 8.422404289245605,
|
|
"kl": 4.848169803619385,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -43597803.921568625,
|
|
"logits/rejected": -42906237.7005988,
|
|
"logps/chosen": -452.0413602941176,
|
|
"logps/rejected": -378.8748128742515,
|
|
"loss": 0.3572,
|
|
"loss/base_kto": 2.8578033447265625,
|
|
"metrics/advantage_var": 8165.21875,
|
|
"rewards/chosen": 0.0707818324269812,
|
|
"rewards/margins": 2.411239406049268,
|
|
"rewards/rejected": -2.340457573622287,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 6.798046588897705,
|
|
"kl": 18.345449447631836,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -43084744.38338658,
|
|
"logits/rejected": -43517958.26299694,
|
|
"logps/chosen": -465.7491513578275,
|
|
"logps/rejected": -387.7618501529052,
|
|
"loss": 0.3413,
|
|
"loss/base_kto": 2.7300755977630615,
|
|
"metrics/advantage_var": 7656.23583984375,
|
|
"rewards/chosen": 1.2981610282922325,
|
|
"rewards/margins": 2.6447027800721443,
|
|
"rewards/rejected": -1.346541751779912,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 7.999556064605713,
|
|
"kl": 21.403776168823242,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -45108135.69742814,
|
|
"logits/rejected": -44685381.47980614,
|
|
"logps/chosen": -470.1483547655068,
|
|
"logps/rejected": -384.3038166397415,
|
|
"loss": 0.3412,
|
|
"loss/base_kto": 2.729660749435425,
|
|
"metrics/advantage_var": 8261.6982421875,
|
|
"rewards/chosen": 1.6109587745839637,
|
|
"rewards/margins": 2.6943660271776126,
|
|
"rewards/rejected": -1.0834072525936491,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 9.10338306427002,
|
|
"kl": 17.625547409057617,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -45012449.88235294,
|
|
"logits/rejected": -44789968.47904192,
|
|
"logps/chosen": -475.2092524509804,
|
|
"logps/rejected": -393.1947979041916,
|
|
"loss": 0.3477,
|
|
"loss/base_kto": 2.7819411754608154,
|
|
"metrics/advantage_var": 9123.5576171875,
|
|
"rewards/chosen": 1.1569894030203227,
|
|
"rewards/margins": 2.65279250506818,
|
|
"rewards/rejected": -1.4958031020478575,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 7.185104846954346,
|
|
"kl": 39.15957260131836,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -45259307.60883281,
|
|
"logits/rejected": -45467172.45820434,
|
|
"logps/chosen": -475.2610410094637,
|
|
"logps/rejected": -367.40707236842104,
|
|
"loss": 0.3101,
|
|
"loss/base_kto": 2.4805924892425537,
|
|
"metrics/advantage_var": 9365.8955078125,
|
|
"rewards/chosen": 2.171943929293178,
|
|
"rewards/margins": 3.200332457556046,
|
|
"rewards/rejected": -1.0283885282628678,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 8.035014152526855,
|
|
"kl": 32.20772171020508,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -46044747.64541213,
|
|
"logits/rejected": -45784483.566718996,
|
|
"logps/chosen": -448.41854587869364,
|
|
"logps/rejected": -384.4145898744113,
|
|
"loss": 0.3403,
|
|
"loss/base_kto": 2.7223238945007324,
|
|
"metrics/advantage_var": 8992.5966796875,
|
|
"rewards/chosen": 1.9568911444158243,
|
|
"rewards/margins": 2.6815834007504495,
|
|
"rewards/rejected": -0.7246922563346252,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 8.956879615783691,
|
|
"kl": 37.35211181640625,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -46692901.62300319,
|
|
"logits/rejected": -47316591.16819572,
|
|
"logps/chosen": -488.006589456869,
|
|
"logps/rejected": -369.25248470948014,
|
|
"loss": 0.3429,
|
|
"loss/base_kto": 2.7429778575897217,
|
|
"metrics/advantage_var": 9564.0224609375,
|
|
"rewards/chosen": 1.9750145860373403,
|
|
"rewards/margins": 2.805674971495817,
|
|
"rewards/rejected": -0.8306603854584766,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.948576927185059,
|
|
"kl": 21.236835479736328,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -45227771.52305246,
|
|
"logits/rejected": -44795060.89093702,
|
|
"logps/chosen": -474.6163056438792,
|
|
"logps/rejected": -375.4060579877112,
|
|
"loss": 0.3341,
|
|
"loss/base_kto": 2.6726529598236084,
|
|
"metrics/advantage_var": 10510.228515625,
|
|
"rewards/chosen": 1.8970902629421702,
|
|
"rewards/margins": 3.1445029723555726,
|
|
"rewards/rejected": -1.2474127094134024,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 7.4264655113220215,
|
|
"kl": 23.38250160217285,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -49304830.78288431,
|
|
"logits/rejected": -48731349.0046225,
|
|
"logps/chosen": -486.996236133122,
|
|
"logps/rejected": -385.19794876733437,
|
|
"loss": 0.3096,
|
|
"loss/base_kto": 2.4769375324249268,
|
|
"metrics/advantage_var": 10881.8251953125,
|
|
"rewards/chosen": 1.862114714367076,
|
|
"rewards/margins": 3.4746163319397843,
|
|
"rewards/rejected": -1.612501617572708,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 8.825889587402344,
|
|
"kl": 29.91910171508789,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -46931557.75394322,
|
|
"logits/rejected": -47508308.80495356,
|
|
"logps/chosen": -453.5905954258675,
|
|
"logps/rejected": -374.0875096749226,
|
|
"loss": 0.3183,
|
|
"loss/base_kto": 2.546102285385132,
|
|
"metrics/advantage_var": 11163.6689453125,
|
|
"rewards/chosen": 1.95512703014097,
|
|
"rewards/margins": 3.2599436865784215,
|
|
"rewards/rejected": -1.3048166564374517,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 9.196476936340332,
|
|
"kl": 32.796630859375,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -47745985.30612245,
|
|
"logits/rejected": -47962675.75738725,
|
|
"logps/chosen": -449.0790816326531,
|
|
"logps/rejected": -392.8316728227061,
|
|
"loss": 0.3159,
|
|
"loss/base_kto": 2.527033567428589,
|
|
"metrics/advantage_var": 9498.0673828125,
|
|
"rewards/chosen": 2.36832139331093,
|
|
"rewards/margins": 3.255964498445553,
|
|
"rewards/rejected": -0.8876431051346229,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 8.986213684082031,
|
|
"kl": 26.47309684753418,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -47960802.30721003,
|
|
"logits/rejected": -48642245.78193147,
|
|
"logps/chosen": -485.68696120689657,
|
|
"logps/rejected": -394.78448695482865,
|
|
"loss": 0.3287,
|
|
"loss/base_kto": 2.6298370361328125,
|
|
"metrics/advantage_var": 10601.69921875,
|
|
"rewards/chosen": 1.781288649221199,
|
|
"rewards/margins": 3.213277897146384,
|
|
"rewards/rejected": -1.431989247925185,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 7.825324058532715,
|
|
"kl": 47.7818717956543,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -47315026.17283951,
|
|
"logits/rejected": -47319033.51898734,
|
|
"logps/chosen": -487.53462577160496,
|
|
"logps/rejected": -359.762089596519,
|
|
"loss": 0.319,
|
|
"loss/base_kto": 2.551891803741455,
|
|
"metrics/advantage_var": 9312.2255859375,
|
|
"rewards/chosen": 2.5791514832296487,
|
|
"rewards/margins": 3.1936893621111757,
|
|
"rewards/rejected": -0.6145378788815269,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 6.205320835113525,
|
|
"kl": 39.29474639892578,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -46901672.36036036,
|
|
"logits/rejected": -47044868.169381104,
|
|
"logps/chosen": -454.5004222972973,
|
|
"logps/rejected": -378.09731270358304,
|
|
"loss": 0.3192,
|
|
"loss/base_kto": 2.5539467334747314,
|
|
"metrics/advantage_var": 10707.734375,
|
|
"rewards/chosen": 2.507337965406813,
|
|
"rewards/margins": 3.290744872280286,
|
|
"rewards/rejected": -0.7834069068734731,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 5.915700435638428,
|
|
"kl": 25.44234848022461,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -48082336.28486647,
|
|
"logits/rejected": -47447279.94719472,
|
|
"logps/chosen": -454.9650871661721,
|
|
"logps/rejected": -394.4505466171617,
|
|
"loss": 0.3307,
|
|
"loss/base_kto": 2.645360231399536,
|
|
"metrics/advantage_var": 10400.4521484375,
|
|
"rewards/chosen": 1.958246338615078,
|
|
"rewards/margins": 3.180263028809488,
|
|
"rewards/rejected": -1.22201669019441,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 8.638898849487305,
|
|
"kl": 44.32819747924805,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -48419350.92537314,
|
|
"logits/rejected": -47859839.58032787,
|
|
"logps/chosen": -463.88768656716417,
|
|
"logps/rejected": -381.01787909836065,
|
|
"loss": 0.3098,
|
|
"loss/base_kto": 2.478761672973633,
|
|
"metrics/advantage_var": 10415.7021484375,
|
|
"rewards/chosen": 2.7349866269239738,
|
|
"rewards/margins": 3.2418057547415966,
|
|
"rewards/rejected": -0.506819127817623,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 8.295904159545898,
|
|
"kl": 15.3886079788208,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -49457854.996865205,
|
|
"logits/rejected": -49579368.47352025,
|
|
"logps/chosen": -491.1561520376176,
|
|
"logps/rejected": -419.11808800623055,
|
|
"loss": 0.3314,
|
|
"loss/base_kto": 2.651231288909912,
|
|
"metrics/advantage_var": 13674.0029296875,
|
|
"rewards/chosen": 1.5671528305005877,
|
|
"rewards/margins": 3.690570289673582,
|
|
"rewards/rejected": -2.1234174591729946,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 8.249643325805664,
|
|
"kl": 36.11222457885742,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -47823306.27160494,
|
|
"logits/rejected": -48412691.44303797,
|
|
"logps/chosen": -468.3174672067901,
|
|
"logps/rejected": -374.06002768987344,
|
|
"loss": 0.3015,
|
|
"loss/base_kto": 2.412252426147461,
|
|
"metrics/advantage_var": 12290.0556640625,
|
|
"rewards/chosen": 2.395194536373939,
|
|
"rewards/margins": 3.7916094742262434,
|
|
"rewards/rejected": -1.396414937852304,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 8.483682632446289,
|
|
"kl": 19.95432472229004,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -49081974.15384615,
|
|
"logits/rejected": -47577837.26829268,
|
|
"logps/chosen": -472.70347556089746,
|
|
"logps/rejected": -404.7442835365854,
|
|
"loss": 0.3209,
|
|
"loss/base_kto": 2.5670926570892334,
|
|
"metrics/advantage_var": 13209.6298828125,
|
|
"rewards/chosen": 1.3851619622646234,
|
|
"rewards/margins": 3.574410582274627,
|
|
"rewards/rejected": -2.189248620010004,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 6.4333648681640625,
|
|
"kl": 21.872238159179688,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -47805700.13509545,
|
|
"logits/rejected": -47969828.75459099,
|
|
"logps/chosen": -473.998669236417,
|
|
"logps/rejected": -388.78177170283806,
|
|
"loss": 0.3352,
|
|
"loss/base_kto": 2.6814286708831787,
|
|
"metrics/advantage_var": 14399.0498046875,
|
|
"rewards/chosen": 1.7137072951082966,
|
|
"rewards/margins": 3.7160327447509305,
|
|
"rewards/rejected": -2.0023254496426337,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 7.0378828048706055,
|
|
"kl": 44.050533294677734,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -47909045.75594295,
|
|
"logits/rejected": -47041594.55950541,
|
|
"logps/chosen": -446.0818145800317,
|
|
"logps/rejected": -369.4335394126739,
|
|
"loss": 0.3069,
|
|
"loss/base_kto": 2.455458164215088,
|
|
"metrics/advantage_var": 12130.0087890625,
|
|
"rewards/chosen": 2.7584470334786055,
|
|
"rewards/margins": 3.6354136484745,
|
|
"rewards/rejected": -0.8769666149958946,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 7.128819942474365,
|
|
"kl": 8.479227066040039,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -49175786.102643855,
|
|
"logits/rejected": -49501764.320251174,
|
|
"logps/chosen": -472.32008164852255,
|
|
"logps/rejected": -406.70111361852435,
|
|
"loss": 0.2763,
|
|
"loss/base_kto": 2.2100369930267334,
|
|
"metrics/advantage_var": 14933.9052734375,
|
|
"rewards/chosen": 1.5404610878572123,
|
|
"rewards/margins": 4.540625509094458,
|
|
"rewards/rejected": -3.000164421237245,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 6.075253486633301,
|
|
"kl": 22.863582611083984,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -48807158.518518515,
|
|
"logits/rejected": -49843264.81012658,
|
|
"logps/chosen": -472.3656925154321,
|
|
"logps/rejected": -378.02037183544303,
|
|
"loss": 0.2606,
|
|
"loss/base_kto": 2.084904670715332,
|
|
"metrics/advantage_var": 14587.2138671875,
|
|
"rewards/chosen": 2.5730581401306907,
|
|
"rewards/margins": 4.949998130983143,
|
|
"rewards/rejected": -2.3769399908524527,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 8.257942199707031,
|
|
"kl": 24.318510055541992,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -50012448.0,
|
|
"logits/rejected": -49810028.8,
|
|
"logps/chosen": -448.85087890625,
|
|
"logps/rejected": -413.91298828125,
|
|
"loss": 0.2571,
|
|
"loss/base_kto": 2.0571820735931396,
|
|
"metrics/advantage_var": 14592.2490234375,
|
|
"rewards/chosen": 2.8325862884521484,
|
|
"rewards/margins": 4.869905662536621,
|
|
"rewards/rejected": -2.037319374084473,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 9.605294227600098,
|
|
"kl": 9.2108793258667,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -49713686.5408805,
|
|
"logits/rejected": -50184201.54037267,
|
|
"logps/chosen": -483.2286261792453,
|
|
"logps/rejected": -399.96535326086956,
|
|
"loss": 0.2791,
|
|
"loss/base_kto": 2.2326481342315674,
|
|
"metrics/advantage_var": 14464.5341796875,
|
|
"rewards/chosen": 2.036675027331466,
|
|
"rewards/margins": 4.535472141922887,
|
|
"rewards/rejected": -2.498797114591421,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 8.508650779724121,
|
|
"kl": 23.149219512939453,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -48690886.37059724,
|
|
"logits/rejected": -49778936.24242424,
|
|
"logps/chosen": -426.35892036753444,
|
|
"logps/rejected": -397.50652910685807,
|
|
"loss": 0.2715,
|
|
"loss/base_kto": 2.1716575622558594,
|
|
"metrics/advantage_var": 13795.7470703125,
|
|
"rewards/chosen": 2.4378465824799003,
|
|
"rewards/margins": 4.29703390180805,
|
|
"rewards/rejected": -1.8591873193281498,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 9.159284591674805,
|
|
"kl": 8.661831855773926,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -49975672.1632653,
|
|
"logits/rejected": -49788434.31415241,
|
|
"logps/chosen": -456.75132456828885,
|
|
"logps/rejected": -394.9606580482115,
|
|
"loss": 0.271,
|
|
"loss/base_kto": 2.1682369709014893,
|
|
"metrics/advantage_var": 14302.1611328125,
|
|
"rewards/chosen": 1.9851262019230769,
|
|
"rewards/margins": 4.683231321358341,
|
|
"rewards/rejected": -2.6981051194352643,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 8.35326099395752,
|
|
"kl": 20.57655906677246,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -48238076.18479881,
|
|
"logits/rejected": -48799912.14449918,
|
|
"logps/chosen": -457.77868852459017,
|
|
"logps/rejected": -390.96544027093597,
|
|
"loss": 0.262,
|
|
"loss/base_kto": 2.096139669418335,
|
|
"metrics/advantage_var": 14034.4755859375,
|
|
"rewards/chosen": 2.3559140974292103,
|
|
"rewards/margins": 4.828909097557494,
|
|
"rewards/rejected": -2.472995000128284,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 7.199108123779297,
|
|
"kl": 16.808950424194336,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -50340605.72106825,
|
|
"logits/rejected": -49781594.40264026,
|
|
"logps/chosen": -470.5954191394659,
|
|
"logps/rejected": -382.4571730610561,
|
|
"loss": 0.2585,
|
|
"loss/base_kto": 2.068336248397827,
|
|
"metrics/advantage_var": 13719.765625,
|
|
"rewards/chosen": 2.7827637443202895,
|
|
"rewards/margins": 4.609518172692814,
|
|
"rewards/rejected": -1.8267544283725248,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 7.581564426422119,
|
|
"kl": 20.133108139038086,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -50324385.42675159,
|
|
"logits/rejected": -49409947.48466258,
|
|
"logps/chosen": -476.8251393312102,
|
|
"logps/rejected": -413.8957055214724,
|
|
"loss": 0.2575,
|
|
"loss/base_kto": 2.059846878051758,
|
|
"metrics/advantage_var": 14377.228515625,
|
|
"rewards/chosen": 2.26127595354797,
|
|
"rewards/margins": 5.040482024711217,
|
|
"rewards/rejected": -2.7792060711632476,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 6.550929069519043,
|
|
"kl": 18.76786231994629,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -50265547.880239524,
|
|
"logits/rejected": -51629383.94771242,
|
|
"logps/chosen": -467.468001497006,
|
|
"logps/rejected": -375.1931168300654,
|
|
"loss": 0.2578,
|
|
"loss/base_kto": 2.0624940395355225,
|
|
"metrics/advantage_var": 13318.0908203125,
|
|
"rewards/chosen": 2.5435257414857784,
|
|
"rewards/margins": 4.6974037477664154,
|
|
"rewards/rejected": -2.153878006280637,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 7.907017230987549,
|
|
"kl": 26.3935604095459,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -49740227.99391172,
|
|
"logits/rejected": -49904444.40449438,
|
|
"logps/chosen": -470.5619292237443,
|
|
"logps/rejected": -384.57641954253614,
|
|
"loss": 0.2526,
|
|
"loss/base_kto": 2.020489454269409,
|
|
"metrics/advantage_var": 14414.7890625,
|
|
"rewards/chosen": 2.637302903824201,
|
|
"rewards/margins": 4.767513523118442,
|
|
"rewards/rejected": -2.1302106192942416,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 11.054191589355469,
|
|
"kl": 25.65165138244629,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -49833681.7833595,
|
|
"logits/rejected": -48465050.47589424,
|
|
"logps/chosen": -455.166601255887,
|
|
"logps/rejected": -377.271967340591,
|
|
"loss": 0.2636,
|
|
"loss/base_kto": 2.1088874340057373,
|
|
"metrics/advantage_var": 15088.5439453125,
|
|
"rewards/chosen": 2.379607244039443,
|
|
"rewards/margins": 4.768339869428246,
|
|
"rewards/rejected": -2.3887326253888026,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 8.053932189941406,
|
|
"kl": 21.888080596923828,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -49561057.11746032,
|
|
"logits/rejected": -49377758.916923076,
|
|
"logps/chosen": -458.37624007936506,
|
|
"logps/rejected": -390.8678125,
|
|
"loss": 0.2553,
|
|
"loss/base_kto": 2.0426578521728516,
|
|
"metrics/advantage_var": 13139.5380859375,
|
|
"rewards/chosen": 2.727308679005456,
|
|
"rewards/margins": 4.735687960856417,
|
|
"rewards/rejected": -2.0083792818509614,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 7.707869529724121,
|
|
"kl": 17.26511001586914,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -52352816.231884055,
|
|
"logits/rejected": -50427248.26707132,
|
|
"logps/chosen": -503.0871578099839,
|
|
"logps/rejected": -401.6888277693475,
|
|
"loss": 0.2615,
|
|
"loss/base_kto": 2.0917253494262695,
|
|
"metrics/advantage_var": 13824.634765625,
|
|
"rewards/chosen": 2.185942374949678,
|
|
"rewards/margins": 4.890966411131393,
|
|
"rewards/rejected": -2.705024036181715,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 8.309268951416016,
|
|
"kl": 18.66203498840332,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -49922970.10114192,
|
|
"logits/rejected": -50169263.40029985,
|
|
"logps/chosen": -443.8905995106036,
|
|
"logps/rejected": -399.65423538230885,
|
|
"loss": 0.2728,
|
|
"loss/base_kto": 2.182373523712158,
|
|
"metrics/advantage_var": 13755.0751953125,
|
|
"rewards/chosen": 2.116479173570045,
|
|
"rewards/margins": 4.563443923616896,
|
|
"rewards/rejected": -2.4469647500468517,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 8.86742115020752,
|
|
"kl": 19.643617630004883,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -49900602.32911392,
|
|
"logits/rejected": -50785434.86419753,
|
|
"logps/chosen": -470.07352650316454,
|
|
"logps/rejected": -404.669319058642,
|
|
"loss": 0.2568,
|
|
"loss/base_kto": 2.0543911457061768,
|
|
"metrics/advantage_var": 13535.3701171875,
|
|
"rewards/chosen": 2.5262949496884888,
|
|
"rewards/margins": 4.746610388418234,
|
|
"rewards/rejected": -2.2203154387297452,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 8.328465461730957,
|
|
"kl": 24.861005783081055,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -48342120.818897635,
|
|
"logits/rejected": -49282109.91627907,
|
|
"logps/chosen": -436.20187007874017,
|
|
"logps/rejected": -389.5715600775194,
|
|
"loss": 0.2647,
|
|
"loss/base_kto": 2.1176109313964844,
|
|
"metrics/advantage_var": 13335.603515625,
|
|
"rewards/chosen": 2.118761534202756,
|
|
"rewards/margins": 4.478910024694519,
|
|
"rewards/rejected": -2.360148490491764,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 8.225176811218262,
|
|
"kl": 16.95220375061035,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -48880521.66409861,
|
|
"logits/rejected": -48887918.3518225,
|
|
"logps/chosen": -454.7324730354391,
|
|
"logps/rejected": -381.3492224643423,
|
|
"loss": 0.2655,
|
|
"loss/base_kto": 2.123788833618164,
|
|
"metrics/advantage_var": 13949.244140625,
|
|
"rewards/chosen": 2.2439794319987483,
|
|
"rewards/margins": 4.5772557604951825,
|
|
"rewards/rejected": -2.333276328496434,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 8.387892723083496,
|
|
"kl": 21.560420989990234,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -51606187.77309562,
|
|
"logits/rejected": -51349188.92307692,
|
|
"logps/chosen": -462.6977816045381,
|
|
"logps/rejected": -391.9108691553545,
|
|
"loss": 0.2572,
|
|
"loss/base_kto": 2.0576674938201904,
|
|
"metrics/advantage_var": 14644.9970703125,
|
|
"rewards/chosen": 2.266873005723258,
|
|
"rewards/margins": 4.8239275282534235,
|
|
"rewards/rejected": -2.5570545225301657,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 7.858390808105469,
|
|
"kl": 15.610109329223633,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -50757154.027820714,
|
|
"logits/rejected": -51382676.423380725,
|
|
"logps/chosen": -465.051970633694,
|
|
"logps/rejected": -391.02142575039494,
|
|
"loss": 0.2454,
|
|
"loss/base_kto": 1.9634945392608643,
|
|
"metrics/advantage_var": 15700.1689453125,
|
|
"rewards/chosen": 2.402788636857612,
|
|
"rewards/margins": 5.1408103187049665,
|
|
"rewards/rejected": -2.738021681847354,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 9.813386917114258,
|
|
"kl": 25.023880004882812,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -50744911.50310559,
|
|
"logits/rejected": -52875898.364779875,
|
|
"logps/chosen": -453.86655667701865,
|
|
"logps/rejected": -409.5614435927673,
|
|
"loss": 0.2585,
|
|
"loss/base_kto": 2.068122625350952,
|
|
"metrics/advantage_var": 14208.1142578125,
|
|
"rewards/chosen": 2.5509571525621118,
|
|
"rewards/margins": 4.723011035472882,
|
|
"rewards/rejected": -2.1720538829107703,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 9.781533241271973,
|
|
"kl": 18.06972312927246,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -50727252.22149837,
|
|
"logits/rejected": -50698756.61261261,
|
|
"logps/chosen": -470.33092426710095,
|
|
"logps/rejected": -399.44237987987987,
|
|
"loss": 0.2447,
|
|
"loss/base_kto": 1.9572668075561523,
|
|
"metrics/advantage_var": 14845.8779296875,
|
|
"rewards/chosen": 2.0454725830873373,
|
|
"rewards/margins": 5.094310972257757,
|
|
"rewards/rejected": -3.0488383891704203,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 7.7680559158325195,
|
|
"kl": 25.265140533447266,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -50514493.94180705,
|
|
"logits/rejected": -51570998.303030305,
|
|
"logps/chosen": -452.76732388973966,
|
|
"logps/rejected": -400.32675438596493,
|
|
"loss": 0.2464,
|
|
"loss/base_kto": 1.9708786010742188,
|
|
"metrics/advantage_var": 14639.181640625,
|
|
"rewards/chosen": 2.538979686602699,
|
|
"rewards/margins": 4.9464379296001075,
|
|
"rewards/rejected": -2.4074582429974085,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 9.849201202392578,
|
|
"kl": 30.48682975769043,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -51322437.451104105,
|
|
"logits/rejected": -52218033.53560372,
|
|
"logps/chosen": -479.2074625394322,
|
|
"logps/rejected": -386.64439821981426,
|
|
"loss": 0.2403,
|
|
"loss/base_kto": 1.9221832752227783,
|
|
"metrics/advantage_var": 13286.900390625,
|
|
"rewards/chosen": 3.028706162509858,
|
|
"rewards/margins": 4.989388939938264,
|
|
"rewards/rejected": -1.9606827774284055,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 13.191734313964844,
|
|
"kl": 14.286864280700684,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -51283517.44,
|
|
"logits/rejected": -50448852.114285715,
|
|
"logps/chosen": -453.09653846153844,
|
|
"logps/rejected": -413.8794642857143,
|
|
"loss": 0.2527,
|
|
"loss/base_kto": 2.0218100547790527,
|
|
"metrics/advantage_var": 14516.5849609375,
|
|
"rewards/chosen": 2.2963213641826923,
|
|
"rewards/margins": 4.983963198260073,
|
|
"rewards/rejected": -2.687641834077381,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 8.107991218566895,
|
|
"kl": 22.3076114654541,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -51279175.19526627,
|
|
"logits/rejected": -50293064.90066225,
|
|
"logps/chosen": -459.0122503698225,
|
|
"logps/rejected": -396.8199503311258,
|
|
"loss": 0.2464,
|
|
"loss/base_kto": 1.97113037109375,
|
|
"metrics/advantage_var": 14052.2333984375,
|
|
"rewards/chosen": 2.767571985368898,
|
|
"rewards/margins": 4.981228783472686,
|
|
"rewards/rejected": -2.2136567981037873,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 7.876888751983643,
|
|
"kl": 23.63214683532715,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -51858099.57585645,
|
|
"logits/rejected": -50774759.05247376,
|
|
"logps/chosen": -460.11154159869494,
|
|
"logps/rejected": -404.99058283358323,
|
|
"loss": 0.2449,
|
|
"loss/base_kto": 1.9594535827636719,
|
|
"metrics/advantage_var": 14345.2470703125,
|
|
"rewards/chosen": 2.6476365753721454,
|
|
"rewards/margins": 5.056325968468566,
|
|
"rewards/rejected": -2.4086893930964206,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 8.337098121643066,
|
|
"kl": 19.482269287109375,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -49769627.5443038,
|
|
"logits/rejected": -50932467.358024694,
|
|
"logps/chosen": -467.47542523734177,
|
|
"logps/rejected": -385.0291521990741,
|
|
"loss": 0.2689,
|
|
"loss/base_kto": 2.151118040084839,
|
|
"metrics/advantage_var": 15071.4501953125,
|
|
"rewards/chosen": 2.0300922635235366,
|
|
"rewards/margins": 4.762695736951391,
|
|
"rewards/rejected": -2.732603473427855,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 8.486111640930176,
|
|
"kl": 18.892194747924805,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -51199834.63354037,
|
|
"logits/rejected": -51612607.597484276,
|
|
"logps/chosen": -451.39314829192546,
|
|
"logps/rejected": -402.25515919811323,
|
|
"loss": 0.2552,
|
|
"loss/base_kto": 2.041901111602783,
|
|
"metrics/advantage_var": 13988.0927734375,
|
|
"rewards/chosen": 2.632206318541343,
|
|
"rewards/margins": 4.877813322766972,
|
|
"rewards/rejected": -2.245607004225629,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 5.810118198394775,
|
|
"kl": 18.350488662719727,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -50581396.377125196,
|
|
"logits/rejected": -51894417.59241706,
|
|
"logps/chosen": -479.9468701700155,
|
|
"logps/rejected": -409.77739928909955,
|
|
"loss": 0.2533,
|
|
"loss/base_kto": 2.0267603397369385,
|
|
"metrics/advantage_var": 15810.369140625,
|
|
"rewards/chosen": 2.52049875627898,
|
|
"rewards/margins": 5.100344118455618,
|
|
"rewards/rejected": -2.579845362176639,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 8.731647491455078,
|
|
"kl": 19.821945190429688,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -50769305.271268055,
|
|
"logits/rejected": -50592647.98782344,
|
|
"logps/chosen": -472.839386035313,
|
|
"logps/rejected": -380.9224457762557,
|
|
"loss": 0.2307,
|
|
"loss/base_kto": 1.8458435535430908,
|
|
"metrics/advantage_var": 15380.0625,
|
|
"rewards/chosen": 2.470528542899779,
|
|
"rewards/margins": 5.446726689957903,
|
|
"rewards/rejected": -2.976198147058124,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 12.344839096069336,
|
|
"kl": 20.61775016784668,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -51296505.91442155,
|
|
"logits/rejected": -52072858.23112481,
|
|
"logps/chosen": -460.2765946909667,
|
|
"logps/rejected": -402.63467835130973,
|
|
"loss": 0.262,
|
|
"loss/base_kto": 2.095961093902588,
|
|
"metrics/advantage_var": 14151.8955078125,
|
|
"rewards/chosen": 2.6214164334885104,
|
|
"rewards/margins": 4.732221668706153,
|
|
"rewards/rejected": -2.1108052352176427,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 10.104900360107422,
|
|
"kl": 21.741413116455078,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -49976328.16586922,
|
|
"logits/rejected": -50593735.5467075,
|
|
"logps/chosen": -460.589812599681,
|
|
"logps/rejected": -404.05915007656967,
|
|
"loss": 0.2625,
|
|
"loss/base_kto": 2.1003079414367676,
|
|
"metrics/advantage_var": 14812.681640625,
|
|
"rewards/chosen": 2.3763505610172446,
|
|
"rewards/margins": 4.787454585112766,
|
|
"rewards/rejected": -2.4111040240955206,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 9.464112281799316,
|
|
"kl": 26.843799591064453,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -51003599.06477093,
|
|
"logits/rejected": -50613699.06646059,
|
|
"logps/chosen": -435.2705371248025,
|
|
"logps/rejected": -421.655477202473,
|
|
"loss": 0.2546,
|
|
"loss/base_kto": 2.036956548690796,
|
|
"metrics/advantage_var": 13646.5810546875,
|
|
"rewards/chosen": 2.445724800676343,
|
|
"rewards/margins": 4.735024837806849,
|
|
"rewards/rejected": -2.289300037130506,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 8.945937156677246,
|
|
"kl": 18.546194076538086,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -51285871.35794543,
|
|
"logits/rejected": -50765618.264840186,
|
|
"logps/chosen": -477.2027989566613,
|
|
"logps/rejected": -391.77960901826486,
|
|
"loss": 0.2609,
|
|
"loss/base_kto": 2.0872085094451904,
|
|
"metrics/advantage_var": 13895.5595703125,
|
|
"rewards/chosen": 2.4925454814782304,
|
|
"rewards/margins": 4.680363570329067,
|
|
"rewards/rejected": -2.187818088850837,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 8.474348068237305,
|
|
"kl": 16.511445999145508,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -51369294.923317686,
|
|
"logits/rejected": -51402240.79875195,
|
|
"logps/chosen": -462.3911384976526,
|
|
"logps/rejected": -414.3921606864275,
|
|
"loss": 0.2603,
|
|
"loss/base_kto": 2.0823142528533936,
|
|
"metrics/advantage_var": 14479.283203125,
|
|
"rewards/chosen": 2.201315914148083,
|
|
"rewards/margins": 4.8964726364326,
|
|
"rewards/rejected": -2.6951567222845165,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 7.912738800048828,
|
|
"kl": 25.281246185302734,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -52288512.0,
|
|
"logits/rejected": -53545018.32911392,
|
|
"logps/chosen": -473.1086516203704,
|
|
"logps/rejected": -385.10087025316454,
|
|
"loss": 0.2629,
|
|
"loss/base_kto": 2.1033740043640137,
|
|
"metrics/advantage_var": 12775.525390625,
|
|
"rewards/chosen": 2.3238201376832563,
|
|
"rewards/margins": 4.495245486278984,
|
|
"rewards/rejected": -2.171425348595728,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 8.972837448120117,
|
|
"kl": 24.49571418762207,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -50361548.15698587,
|
|
"logits/rejected": -51107225.281493,
|
|
"logps/chosen": -474.48606750392463,
|
|
"logps/rejected": -402.76513899688956,
|
|
"loss": 0.2375,
|
|
"loss/base_kto": 1.9002567529678345,
|
|
"metrics/advantage_var": 14855.8740234375,
|
|
"rewards/chosen": 2.919538033813285,
|
|
"rewards/margins": 5.343848744714334,
|
|
"rewards/rejected": -2.4243107109010498,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 7.34293794631958,
|
|
"kl": 18.954492568969727,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -50771734.56534954,
|
|
"logits/rejected": -51986418.787096776,
|
|
"logps/chosen": -466.31449468085106,
|
|
"logps/rejected": -376.00982862903226,
|
|
"loss": 0.2148,
|
|
"loss/base_kto": 1.7182190418243408,
|
|
"metrics/advantage_var": 15409.087890625,
|
|
"rewards/chosen": 2.7397511027141905,
|
|
"rewards/margins": 5.701999283472759,
|
|
"rewards/rejected": -2.9622481807585683,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 5.555119037628174,
|
|
"kl": 14.898538589477539,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -50034223.87321711,
|
|
"logits/rejected": -50908884.21571649,
|
|
"logps/chosen": -441.70032686212363,
|
|
"logps/rejected": -415.6199441448382,
|
|
"loss": 0.2126,
|
|
"loss/base_kto": 1.7008146047592163,
|
|
"metrics/advantage_var": 13632.4970703125,
|
|
"rewards/chosen": 2.8393784899341323,
|
|
"rewards/margins": 5.480414842435577,
|
|
"rewards/rejected": -2.6410363525014446,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 7.122386455535889,
|
|
"kl": 17.392332077026367,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -52030957.14874816,
|
|
"logits/rejected": -52453399.85357737,
|
|
"logps/chosen": -468.22505522827686,
|
|
"logps/rejected": -391.0408433860233,
|
|
"loss": 0.2097,
|
|
"loss/base_kto": 1.6779264211654663,
|
|
"metrics/advantage_var": 14165.556640625,
|
|
"rewards/chosen": 2.5735638131098124,
|
|
"rewards/margins": 5.60465457330116,
|
|
"rewards/rejected": -3.031090760191348,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 8.374948501586914,
|
|
"kl": 18.164281845092773,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -54053437.923303835,
|
|
"logits/rejected": -53630322.81727575,
|
|
"logps/chosen": -474.5418510324484,
|
|
"logps/rejected": -419.30411648671094,
|
|
"loss": 0.2087,
|
|
"loss/base_kto": 1.6698551177978516,
|
|
"metrics/advantage_var": 13417.6748046875,
|
|
"rewards/chosen": 2.791623095847161,
|
|
"rewards/margins": 5.391831061482543,
|
|
"rewards/rejected": -2.600207965635382,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 9.291142463684082,
|
|
"kl": 19.005338668823242,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -52156198.78787879,
|
|
"logits/rejected": -53144311.741935484,
|
|
"logps/chosen": -463.5467803030303,
|
|
"logps/rejected": -409.1705141129032,
|
|
"loss": 0.2125,
|
|
"loss/base_kto": 1.6997795104980469,
|
|
"metrics/advantage_var": 13266.9892578125,
|
|
"rewards/chosen": 2.6969493519176138,
|
|
"rewards/margins": 5.313385528832937,
|
|
"rewards/rejected": -2.6164361769153226,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 8.366868019104004,
|
|
"kl": 14.552406311035156,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -52351320.3539823,
|
|
"logits/rejected": -52654488.239202656,
|
|
"logps/chosen": -473.98101032448375,
|
|
"logps/rejected": -418.17794850498336,
|
|
"loss": 0.2051,
|
|
"loss/base_kto": 1.6408668756484985,
|
|
"metrics/advantage_var": 16153.1005859375,
|
|
"rewards/chosen": 2.771032462781158,
|
|
"rewards/margins": 5.9631924259897335,
|
|
"rewards/rejected": -3.1921599632085758,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 7.207845211029053,
|
|
"kl": 21.19478416442871,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -52147415.31942215,
|
|
"logits/rejected": -52342840.10958904,
|
|
"logps/chosen": -453.4293739967897,
|
|
"logps/rejected": -401.2667903348554,
|
|
"loss": 0.2149,
|
|
"loss/base_kto": 1.718897819519043,
|
|
"metrics/advantage_var": 14809.8955078125,
|
|
"rewards/chosen": 2.812826435217697,
|
|
"rewards/margins": 5.515159371683165,
|
|
"rewards/rejected": -2.702332936465468,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 7.801844596862793,
|
|
"kl": 19.278751373291016,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -51519835.172628306,
|
|
"logits/rejected": -50894349.66405024,
|
|
"logps/chosen": -465.26360808709177,
|
|
"logps/rejected": -372.2115875196232,
|
|
"loss": 0.2053,
|
|
"loss/base_kto": 1.642542839050293,
|
|
"metrics/advantage_var": 12793.9970703125,
|
|
"rewards/chosen": 2.8394145105341173,
|
|
"rewards/margins": 5.430397711206213,
|
|
"rewards/rejected": -2.590983200672096,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 9.775045394897461,
|
|
"kl": 20.726667404174805,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -51509880.662420385,
|
|
"logits/rejected": -51625760.98159509,
|
|
"logps/chosen": -422.7342257165605,
|
|
"logps/rejected": -416.6057323619632,
|
|
"loss": 0.2219,
|
|
"loss/base_kto": 1.7754486799240112,
|
|
"metrics/advantage_var": 14590.0625,
|
|
"rewards/chosen": 2.514944283066282,
|
|
"rewards/margins": 5.329098597914825,
|
|
"rewards/rejected": -2.814154314848543,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 7.695766925811768,
|
|
"kl": 18.871171951293945,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -51663503.74763407,
|
|
"logits/rejected": -51968903.52941176,
|
|
"logps/chosen": -447.028785488959,
|
|
"logps/rejected": -400.92431791795667,
|
|
"loss": 0.2171,
|
|
"loss/base_kto": 1.7367476224899292,
|
|
"metrics/advantage_var": 15574.3642578125,
|
|
"rewards/chosen": 2.6369784863589314,
|
|
"rewards/margins": 5.576990186968452,
|
|
"rewards/rejected": -2.9400117006095203,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 7.314939022064209,
|
|
"kl": 19.452167510986328,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -52745370.752411574,
|
|
"logits/rejected": -53523259.91489362,
|
|
"logps/chosen": -471.878165192926,
|
|
"logps/rejected": -386.12419262917933,
|
|
"loss": 0.1992,
|
|
"loss/base_kto": 1.593440294265747,
|
|
"metrics/advantage_var": 16043.1455078125,
|
|
"rewards/chosen": 2.777079198904743,
|
|
"rewards/margins": 5.964584393386126,
|
|
"rewards/rejected": -3.187505194481383,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.1406474113464355,
|
|
"kl": 11.1013765335083,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -52880064.68817204,
|
|
"logits/rejected": -53137740.10810811,
|
|
"logps/chosen": -439.2087653609831,
|
|
"logps/rejected": -397.033286963434,
|
|
"loss": 0.2087,
|
|
"loss/base_kto": 1.669519066810608,
|
|
"metrics/advantage_var": 14999.6923828125,
|
|
"rewards/chosen": 2.5516636814756146,
|
|
"rewards/margins": 5.534063238373966,
|
|
"rewards/rejected": -2.9823995568983506,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 6.2611212730407715,
|
|
"kl": 14.011194229125977,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -51661088.90282132,
|
|
"logits/rejected": -50629182.205607474,
|
|
"logps/chosen": -458.0458463949843,
|
|
"logps/rejected": -380.8788454049844,
|
|
"loss": 0.2145,
|
|
"loss/base_kto": 1.716043472290039,
|
|
"metrics/advantage_var": 15494.1279296875,
|
|
"rewards/chosen": 2.860801194528801,
|
|
"rewards/margins": 5.573951551689042,
|
|
"rewards/rejected": -2.7131503571602416,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 9.701900482177734,
|
|
"kl": 15.361149787902832,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -51675350.58243451,
|
|
"logits/rejected": -53768451.65134707,
|
|
"logps/chosen": -491.8400423728813,
|
|
"logps/rejected": -406.3640798335975,
|
|
"loss": 0.2072,
|
|
"loss/base_kto": 1.6577314138412476,
|
|
"metrics/advantage_var": 15171.2412109375,
|
|
"rewards/chosen": 2.656865241898594,
|
|
"rewards/margins": 5.613896104987936,
|
|
"rewards/rejected": -2.9570308630893423,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 6.971599102020264,
|
|
"kl": 22.8885555267334,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -52703075.26530612,
|
|
"logits/rejected": -53073008.46242774,
|
|
"logps/chosen": -465.74192176870747,
|
|
"logps/rejected": -418.8117774566474,
|
|
"loss": 0.2017,
|
|
"loss/base_kto": 1.6138557195663452,
|
|
"metrics/advantage_var": 15352.720703125,
|
|
"rewards/chosen": 2.726814944727891,
|
|
"rewards/margins": 5.802486895933635,
|
|
"rewards/rejected": -3.075671951205744,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 13.588480949401855,
|
|
"kl": 13.888775825500488,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -52804236.0832,
|
|
"logits/rejected": -52442382.461068705,
|
|
"logps/chosen": -480.8757,
|
|
"logps/rejected": -408.2770992366412,
|
|
"loss": 0.2267,
|
|
"loss/base_kto": 1.8138843774795532,
|
|
"metrics/advantage_var": 16470.36328125,
|
|
"rewards/chosen": 2.4228767578125,
|
|
"rewards/margins": 5.583166073771469,
|
|
"rewards/rejected": -3.1602893159589693,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 8.179054260253906,
|
|
"kl": 14.419031143188477,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -51960324.16260163,
|
|
"logits/rejected": -52883646.94135338,
|
|
"logps/chosen": -472.74481707317074,
|
|
"logps/rejected": -375.7159539473684,
|
|
"loss": 0.2233,
|
|
"loss/base_kto": 1.7862293720245361,
|
|
"metrics/advantage_var": 14972.625,
|
|
"rewards/chosen": 2.4842916349085367,
|
|
"rewards/margins": 5.570328259673574,
|
|
"rewards/rejected": -3.0860366247650375,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 7.958951950073242,
|
|
"kl": 17.662368774414062,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -51976939.882896766,
|
|
"logits/rejected": -53508605.56576862,
|
|
"logps/chosen": -453.66833590138674,
|
|
"logps/rejected": -387.3939926703645,
|
|
"loss": 0.2156,
|
|
"loss/base_kto": 1.7250031232833862,
|
|
"metrics/advantage_var": 13712.3671875,
|
|
"rewards/chosen": 2.6780703034716873,
|
|
"rewards/margins": 5.375076117965051,
|
|
"rewards/rejected": -2.6970058144933637,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 6.839268207550049,
|
|
"kl": 19.165042877197266,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -51127809.501466274,
|
|
"logits/rejected": -51394149.03010034,
|
|
"logps/chosen": -457.106900659824,
|
|
"logps/rejected": -380.3718122909699,
|
|
"loss": 0.2113,
|
|
"loss/base_kto": 1.6907148361206055,
|
|
"metrics/advantage_var": 14822.4345703125,
|
|
"rewards/chosen": 2.7296326936514848,
|
|
"rewards/margins": 5.532632030634135,
|
|
"rewards/rejected": -2.8029993369826505,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 8.150351524353027,
|
|
"kl": 15.535231590270996,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -50620527.03614458,
|
|
"logits/rejected": -51386268.25974026,
|
|
"logps/chosen": -465.2983810240964,
|
|
"logps/rejected": -405.39891943993507,
|
|
"loss": 0.2084,
|
|
"loss/base_kto": 1.6675866842269897,
|
|
"metrics/advantage_var": 15500.0830078125,
|
|
"rewards/chosen": 2.8343193329960465,
|
|
"rewards/margins": 5.77271345840189,
|
|
"rewards/rejected": -2.938394125405844,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 7.899462699890137,
|
|
"kl": 23.955821990966797,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -53088461.473684214,
|
|
"logits/rejected": -52787882.666666664,
|
|
"logps/chosen": -452.7037417763158,
|
|
"logps/rejected": -396.98456101190476,
|
|
"loss": 0.2092,
|
|
"loss/base_kto": 1.6736135482788086,
|
|
"metrics/advantage_var": 13926.71875,
|
|
"rewards/chosen": 2.632309361508018,
|
|
"rewards/margins": 5.417852894106604,
|
|
"rewards/rejected": -2.7855435325985862,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 7.126053333282471,
|
|
"kl": 23.701719284057617,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -51701858.126738794,
|
|
"logits/rejected": -52832971.02053712,
|
|
"logps/chosen": -450.41373647604325,
|
|
"logps/rejected": -414.1920912322275,
|
|
"loss": 0.2111,
|
|
"loss/base_kto": 1.6890560388565063,
|
|
"metrics/advantage_var": 14298.392578125,
|
|
"rewards/chosen": 2.787808968073802,
|
|
"rewards/margins": 5.523550075646867,
|
|
"rewards/rejected": -2.7357411075730647,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.15418529510498,
|
|
"kl": 16.745986938476562,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -52566420.463949844,
|
|
"logits/rejected": -52571710.205607474,
|
|
"logps/chosen": -462.2469631661442,
|
|
"logps/rejected": -392.3098958333333,
|
|
"loss": 0.2137,
|
|
"loss/base_kto": 1.7097481489181519,
|
|
"metrics/advantage_var": 15660.7626953125,
|
|
"rewards/chosen": 2.7696104617701804,
|
|
"rewards/margins": 5.616432184353417,
|
|
"rewards/rejected": -2.846821722583236,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 6.503498077392578,
|
|
"kl": 18.828340530395508,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -52250798.98734177,
|
|
"logits/rejected": -52860463.40740741,
|
|
"logps/chosen": -467.80241297468353,
|
|
"logps/rejected": -398.2588493441358,
|
|
"loss": 0.2111,
|
|
"loss/base_kto": 1.6891224384307861,
|
|
"metrics/advantage_var": 14132.4580078125,
|
|
"rewards/chosen": 2.7144435447982596,
|
|
"rewards/margins": 5.519844628662071,
|
|
"rewards/rejected": -2.8054010838638117,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 8.204117774963379,
|
|
"kl": 16.509063720703125,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -53636154.74108527,
|
|
"logits/rejected": -52339868.42204724,
|
|
"logps/chosen": -467.9199612403101,
|
|
"logps/rejected": -396.39783464566926,
|
|
"loss": 0.2051,
|
|
"loss/base_kto": 1.6408218145370483,
|
|
"metrics/advantage_var": 15810.578125,
|
|
"rewards/chosen": 2.7192469870397287,
|
|
"rewards/margins": 5.853811916911775,
|
|
"rewards/rejected": -3.134564929872047,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 9.347222328186035,
|
|
"kl": 23.602487564086914,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -49904713.728,
|
|
"logits/rejected": -52168260.00610687,
|
|
"logps/chosen": -432.02695,
|
|
"logps/rejected": -413.0551526717557,
|
|
"loss": 0.2353,
|
|
"loss/base_kto": 1.8824348449707031,
|
|
"metrics/advantage_var": 15216.1328125,
|
|
"rewards/chosen": 2.405577734375,
|
|
"rewards/margins": 5.20243633408874,
|
|
"rewards/rejected": -2.7968585997137403,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 7.752729892730713,
|
|
"kl": 17.468936920166016,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -52361436.35443038,
|
|
"logits/rejected": -53112332.641975306,
|
|
"logps/chosen": -439.2915842563291,
|
|
"logps/rejected": -390.4963348765432,
|
|
"loss": 0.2215,
|
|
"loss/base_kto": 1.771693468093872,
|
|
"metrics/advantage_var": 14805.1787109375,
|
|
"rewards/chosen": 2.445236012905459,
|
|
"rewards/margins": 5.354064559876909,
|
|
"rewards/rejected": -2.9088285469714505,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.719018459320068,
|
|
"kl": 15.244857788085938,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -53175440.52852853,
|
|
"logits/rejected": -52999641.64169381,
|
|
"logps/chosen": -479.3825075075075,
|
|
"logps/rejected": -403.0092121335505,
|
|
"loss": 0.1956,
|
|
"loss/base_kto": 1.5649467706680298,
|
|
"metrics/advantage_var": 15854.3876953125,
|
|
"rewards/chosen": 2.8091128237612613,
|
|
"rewards/margins": 6.051144654291086,
|
|
"rewards/rejected": -3.242031830529825,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 8.016149520874023,
|
|
"kl": 13.679033279418945,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -51404154.43478261,
|
|
"logits/rejected": -51658449.3081761,
|
|
"logps/chosen": -453.99427406832297,
|
|
"logps/rejected": -410.07323604559747,
|
|
"loss": 0.2189,
|
|
"loss/base_kto": 1.7515852451324463,
|
|
"metrics/advantage_var": 15158.03125,
|
|
"rewards/chosen": 2.611411716627038,
|
|
"rewards/margins": 5.540414586430988,
|
|
"rewards/rejected": -2.9290028698039503,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 46.4044189453125,
|
|
"kl": 13.768518447875977,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -52033393.86750789,
|
|
"logits/rejected": -50642709.3993808,
|
|
"logps/chosen": -457.0539235015773,
|
|
"logps/rejected": -400.0735294117647,
|
|
"loss": 0.2263,
|
|
"loss/base_kto": 1.8101516962051392,
|
|
"metrics/advantage_var": 15886.5654296875,
|
|
"rewards/chosen": 2.604241115437697,
|
|
"rewards/margins": 5.479861671020128,
|
|
"rewards/rejected": -2.8756205555824303,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 6.903535842895508,
|
|
"kl": 17.980924606323242,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -52394683.94936709,
|
|
"logits/rejected": -53570484.14814815,
|
|
"logps/chosen": -457.2773931962025,
|
|
"logps/rejected": -413.1441936728395,
|
|
"loss": 0.2028,
|
|
"loss/base_kto": 1.6224689483642578,
|
|
"metrics/advantage_var": 15232.3779296875,
|
|
"rewards/chosen": 2.61376528196697,
|
|
"rewards/margins": 5.606020222047024,
|
|
"rewards/rejected": -2.992254940080054,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 8.415434837341309,
|
|
"kl": 16.8980770111084,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -52471641.41626794,
|
|
"logits/rejected": -53361188.851454824,
|
|
"logps/chosen": -456.282745215311,
|
|
"logps/rejected": -405.41213629402756,
|
|
"loss": 0.2095,
|
|
"loss/base_kto": 1.6762361526489258,
|
|
"metrics/advantage_var": 15096.353515625,
|
|
"rewards/chosen": 2.530509011787281,
|
|
"rewards/margins": 5.562195511296757,
|
|
"rewards/rejected": -3.0316864995094757,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 10.105056762695312,
|
|
"kl": 16.623977661132812,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -54464518.29185868,
|
|
"logits/rejected": -53723507.17965024,
|
|
"logps/chosen": -462.01612903225805,
|
|
"logps/rejected": -417.56369236883944,
|
|
"loss": 0.2251,
|
|
"loss/base_kto": 1.801042914390564,
|
|
"metrics/advantage_var": 15147.4130859375,
|
|
"rewards/chosen": 2.5503351589501726,
|
|
"rewards/margins": 5.4693309297669455,
|
|
"rewards/rejected": -2.918995770816773,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 11.006617546081543,
|
|
"kl": 18.074350357055664,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -54645943.63525836,
|
|
"logits/rejected": -52249079.76848874,
|
|
"logps/chosen": -481.00949848024317,
|
|
"logps/rejected": -411.04677451768487,
|
|
"loss": 0.2147,
|
|
"loss/base_kto": 1.7173265218734741,
|
|
"metrics/advantage_var": 16321.2412109375,
|
|
"rewards/chosen": 2.7446502407271085,
|
|
"rewards/margins": 5.886056922487056,
|
|
"rewards/rejected": -3.1414066817599475,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 27.938901901245117,
|
|
"kl": 20.423267364501953,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -52610920.41574803,
|
|
"logits/rejected": -51983479.069767445,
|
|
"logps/chosen": -442.2832677165354,
|
|
"logps/rejected": -414.7684108527132,
|
|
"loss": 0.2157,
|
|
"loss/base_kto": 1.7258579730987549,
|
|
"metrics/advantage_var": 14801.845703125,
|
|
"rewards/chosen": 2.5830262672244095,
|
|
"rewards/margins": 5.560088971621212,
|
|
"rewards/rejected": -2.9770627043968023,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 8.298346519470215,
|
|
"kl": 19.364654541015625,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -53583106.08130081,
|
|
"logits/rejected": -54432458.49022556,
|
|
"logps/chosen": -469.030487804878,
|
|
"logps/rejected": -403.2602913533835,
|
|
"loss": 0.2137,
|
|
"loss/base_kto": 1.7098503112792969,
|
|
"metrics/advantage_var": 16005.7880859375,
|
|
"rewards/chosen": 2.6091197440294716,
|
|
"rewards/margins": 5.647543072859358,
|
|
"rewards/rejected": -3.038423328829887,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.369647026062012,
|
|
"kl": 14.8077974319458,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -51136837.89079563,
|
|
"logits/rejected": -51940427.31768388,
|
|
"logps/chosen": -461.8348771450858,
|
|
"logps/rejected": -388.8576144366197,
|
|
"loss": 0.2349,
|
|
"loss/base_kto": 1.8791515827178955,
|
|
"metrics/advantage_var": 14433.1123046875,
|
|
"rewards/chosen": 2.4296322731815523,
|
|
"rewards/margins": 5.145286386664534,
|
|
"rewards/rejected": -2.715654113482981,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 7.4242634773254395,
|
|
"kl": 20.830245971679688,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -51962868.640253566,
|
|
"logits/rejected": -52445777.25731895,
|
|
"logps/chosen": -459.29576069730587,
|
|
"logps/rejected": -395.55602369029276,
|
|
"loss": 0.1987,
|
|
"loss/base_kto": 1.5892614126205444,
|
|
"metrics/advantage_var": 13583.431640625,
|
|
"rewards/chosen": 2.650785467326169,
|
|
"rewards/margins": 5.544644484203095,
|
|
"rewards/rejected": -2.893859016876926,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_loss": 0.27575122105642924,
|
|
"train_runtime": 11063.995,
|
|
"train_samples_per_second": 13.396,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|