2a956c7ec3
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed3 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 13.400735855102539,
|
|
"kl": 20.154539108276367,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -38125000.29806259,
|
|
"logits/rejected": -37537566.6863711,
|
|
"logps/chosen": -463.5167660208644,
|
|
"logps/rejected": -394.78040332512313,
|
|
"loss": 0.5246,
|
|
"loss/base_kto": 3.8404922485351562,
|
|
"loss/total_with_kl": 4.196873664855957,
|
|
"metrics/advantage_var": 214.0430145263672,
|
|
"regularization/advantage_var": 214.0430145263672,
|
|
"regularization/kl": 0.3563816547393799,
|
|
"rewards/chosen": 0.32326160777698865,
|
|
"rewards/margins": 0.1113325119872719,
|
|
"rewards/rejected": 0.21192909578971675,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 21.468503952026367,
|
|
"kl": 5.3923821449279785,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37059050.17704918,
|
|
"logits/rejected": -37883155.104477614,
|
|
"logps/chosen": -484.46352459016396,
|
|
"logps/rejected": -372.7042677238806,
|
|
"loss": 0.524,
|
|
"loss/base_kto": 3.8899757862091064,
|
|
"loss/total_with_kl": 4.192325115203857,
|
|
"metrics/advantage_var": 181.59121704101562,
|
|
"regularization/advantage_var": 181.59121704101562,
|
|
"regularization/kl": 0.30234935879707336,
|
|
"rewards/chosen": 0.01020375548816118,
|
|
"rewards/margins": 0.09601645225821714,
|
|
"rewards/rejected": -0.08581269677005597,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 19.037933349609375,
|
|
"kl": 2.5164599418640137,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36605792.71111111,
|
|
"logits/rejected": -37880494.86769231,
|
|
"logps/chosen": -482.24598214285714,
|
|
"logps/rejected": -347.82774038461537,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.826831817626953,
|
|
"loss/total_with_kl": 4.156479835510254,
|
|
"metrics/advantage_var": 197.98660278320312,
|
|
"regularization/advantage_var": 197.98660278320312,
|
|
"regularization/kl": 0.3296476900577545,
|
|
"rewards/chosen": -0.061350740705217635,
|
|
"rewards/margins": 0.18291101560487855,
|
|
"rewards/rejected": -0.24426175631009617,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 18.15247344970703,
|
|
"kl": 3.9609954357147217,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36251251.4031746,
|
|
"logits/rejected": -38429787.372307695,
|
|
"logps/chosen": -491.85843253968255,
|
|
"logps/rejected": -382.4696394230769,
|
|
"loss": 0.5278,
|
|
"loss/base_kto": 3.88032603263855,
|
|
"loss/total_with_kl": 4.222623825073242,
|
|
"metrics/advantage_var": 205.5841827392578,
|
|
"regularization/advantage_var": 205.5841827392578,
|
|
"regularization/kl": 0.3422977030277252,
|
|
"rewards/chosen": -0.10211856175982763,
|
|
"rewards/margins": 0.09690999330327334,
|
|
"rewards/rejected": -0.19902855506310096,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 14.19953727722168,
|
|
"kl": 2.6970603466033936,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35694232.87203792,
|
|
"logits/rejected": -35838502.77588872,
|
|
"logps/chosen": -497.6655805687204,
|
|
"logps/rejected": -364.4688465996909,
|
|
"loss": 0.5163,
|
|
"loss/base_kto": 3.787245273590088,
|
|
"loss/total_with_kl": 4.130555152893066,
|
|
"metrics/advantage_var": 206.1918182373047,
|
|
"regularization/advantage_var": 206.1918182373047,
|
|
"regularization/kl": 0.3433093726634979,
|
|
"rewards/chosen": 0.02683106794553157,
|
|
"rewards/margins": 0.20959978206025576,
|
|
"rewards/rejected": -0.1827687141147242,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 16.51021385192871,
|
|
"kl": 15.355883598327637,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35762474.22288261,
|
|
"logits/rejected": -37603010.84678748,
|
|
"logps/chosen": -482.34193907875186,
|
|
"logps/rejected": -377.04242174629326,
|
|
"loss": 0.5151,
|
|
"loss/base_kto": 3.736530065536499,
|
|
"loss/total_with_kl": 4.121108531951904,
|
|
"metrics/advantage_var": 230.9782257080078,
|
|
"regularization/advantage_var": 230.9782257080078,
|
|
"regularization/kl": 0.38457876443862915,
|
|
"rewards/chosen": 0.3250251305227294,
|
|
"rewards/margins": 0.2259323987640964,
|
|
"rewards/rejected": 0.09909273175863301,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 12.586821556091309,
|
|
"kl": 9.067645072937012,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35689568.60377359,
|
|
"logits/rejected": -36856714.335403726,
|
|
"logps/chosen": -488.97651336477986,
|
|
"logps/rejected": -407.28008540372673,
|
|
"loss": 0.515,
|
|
"loss/base_kto": 3.7524573802948,
|
|
"loss/total_with_kl": 4.1200852394104,
|
|
"metrics/advantage_var": 220.79763793945312,
|
|
"regularization/advantage_var": 220.79763793945312,
|
|
"regularization/kl": 0.3676280677318573,
|
|
"rewards/chosen": 0.15914545119183618,
|
|
"rewards/margins": 0.23662834405163274,
|
|
"rewards/rejected": -0.07748289285979656,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 13.009500503540039,
|
|
"kl": 8.946215629577637,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36485641.42331288,
|
|
"logits/rejected": -37813169.73248408,
|
|
"logps/chosen": -460.18457630368096,
|
|
"logps/rejected": -385.6954617834395,
|
|
"loss": 0.521,
|
|
"loss/base_kto": 3.7870709896087646,
|
|
"loss/total_with_kl": 4.167824745178223,
|
|
"metrics/advantage_var": 228.68077087402344,
|
|
"regularization/advantage_var": 228.68077087402344,
|
|
"regularization/kl": 0.38075345754623413,
|
|
"rewards/chosen": 0.14907382894878737,
|
|
"rewards/margins": 0.20395135918645904,
|
|
"rewards/rejected": -0.054877530237671675,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 15.541783332824707,
|
|
"kl": 13.2084379196167,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36214550.225080386,
|
|
"logits/rejected": -37451903.61094225,
|
|
"logps/chosen": -497.6577572347267,
|
|
"logps/rejected": -371.0432655775076,
|
|
"loss": 0.522,
|
|
"loss/base_kto": 3.7317333221435547,
|
|
"loss/total_with_kl": 4.175820350646973,
|
|
"metrics/advantage_var": 266.7189025878906,
|
|
"regularization/advantage_var": 266.7189025878906,
|
|
"regularization/kl": 0.4440869987010956,
|
|
"rewards/chosen": 0.2306328433119599,
|
|
"rewards/margins": 0.23384995201817044,
|
|
"rewards/rejected": -0.003217108706210522,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 13.655057907104492,
|
|
"kl": 13.225244522094727,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37435065.32515337,
|
|
"logits/rejected": -37178863.69426752,
|
|
"logps/chosen": -475.14239838957053,
|
|
"logps/rejected": -383.55662818471336,
|
|
"loss": 0.5139,
|
|
"loss/base_kto": 3.7183244228363037,
|
|
"loss/total_with_kl": 4.111454963684082,
|
|
"metrics/advantage_var": 236.11441040039062,
|
|
"regularization/advantage_var": 236.11441040039062,
|
|
"regularization/kl": 0.39313045144081116,
|
|
"rewards/chosen": 0.2551800780501102,
|
|
"rewards/margins": 0.25233012410103844,
|
|
"rewards/rejected": 0.002849953949071799,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.7305908203125,
|
|
"kl": 19.802534103393555,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37822903.0912,
|
|
"logits/rejected": -37016402.46717557,
|
|
"logps/chosen": -486.791,
|
|
"logps/rejected": -354.5992604961832,
|
|
"loss": 0.5101,
|
|
"loss/base_kto": 3.684840440750122,
|
|
"loss/total_with_kl": 4.080587863922119,
|
|
"metrics/advantage_var": 237.68594360351562,
|
|
"regularization/advantage_var": 237.68594360351562,
|
|
"regularization/kl": 0.39574703574180603,
|
|
"rewards/chosen": 0.3682064208984375,
|
|
"rewards/margins": 0.27839331660379896,
|
|
"rewards/rejected": 0.0898131042946386,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 17.98505401611328,
|
|
"kl": 27.593021392822266,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36503874.86862442,
|
|
"logits/rejected": -37995806.33175355,
|
|
"logps/chosen": -463.3840803709428,
|
|
"logps/rejected": -361.9566301342812,
|
|
"loss": 0.5185,
|
|
"loss/base_kto": 3.679857015609741,
|
|
"loss/total_with_kl": 4.147706508636475,
|
|
"metrics/advantage_var": 280.9908752441406,
|
|
"regularization/advantage_var": 280.9908752441406,
|
|
"regularization/kl": 0.46784982085227966,
|
|
"rewards/chosen": 0.4872561046476526,
|
|
"rewards/margins": 0.2739743436107993,
|
|
"rewards/rejected": 0.21328176103685328,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 13.71194839477539,
|
|
"kl": 8.389355659484863,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37835969.11568938,
|
|
"logits/rejected": -39779386.37904468,
|
|
"logps/chosen": -495.0869156101426,
|
|
"logps/rejected": -386.3656346302003,
|
|
"loss": 0.5249,
|
|
"loss/base_kto": 3.7221741676330566,
|
|
"loss/total_with_kl": 4.1990790367126465,
|
|
"metrics/advantage_var": 286.4291687011719,
|
|
"regularization/advantage_var": 286.4291687011719,
|
|
"regularization/kl": 0.4769045412540436,
|
|
"rewards/chosen": 0.12366109566907686,
|
|
"rewards/margins": 0.25988230099397575,
|
|
"rewards/rejected": -0.13622120532489887,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 14.024954795837402,
|
|
"kl": 8.718774795532227,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36703920.10191083,
|
|
"logits/rejected": -37847592.83435583,
|
|
"logps/chosen": -483.0296576433121,
|
|
"logps/rejected": -370.02993194018404,
|
|
"loss": 0.5172,
|
|
"loss/base_kto": 3.712843656539917,
|
|
"loss/total_with_kl": 4.1373419761657715,
|
|
"metrics/advantage_var": 254.9535675048828,
|
|
"regularization/advantage_var": 254.9535675048828,
|
|
"regularization/kl": 0.42449769377708435,
|
|
"rewards/chosen": 0.1800000379039983,
|
|
"rewards/margins": 0.2664519486601153,
|
|
"rewards/rejected": -0.086451910756117,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 14.930802345275879,
|
|
"kl": 6.597970485687256,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37782321.27699531,
|
|
"logits/rejected": -38477721.75975039,
|
|
"logps/chosen": -485.4553012519562,
|
|
"logps/rejected": -377.11095943837756,
|
|
"loss": 0.5147,
|
|
"loss/base_kto": 3.685213804244995,
|
|
"loss/total_with_kl": 4.11769962310791,
|
|
"metrics/advantage_var": 259.7512512207031,
|
|
"regularization/advantage_var": 259.7512512207031,
|
|
"regularization/kl": 0.43248581886291504,
|
|
"rewards/chosen": 0.13042405774709018,
|
|
"rewards/margins": 0.30123039500546334,
|
|
"rewards/rejected": -0.17080633725837316,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 15.397305488586426,
|
|
"kl": 14.572275161743164,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35922289.00900901,
|
|
"logits/rejected": -37610392.59934854,
|
|
"logps/chosen": -482.9690315315315,
|
|
"logps/rejected": -346.9250305374593,
|
|
"loss": 0.5085,
|
|
"loss/base_kto": 3.6896419525146484,
|
|
"loss/total_with_kl": 4.067927360534668,
|
|
"metrics/advantage_var": 227.1981964111328,
|
|
"regularization/advantage_var": 227.1981964111328,
|
|
"regularization/kl": 0.3782850205898285,
|
|
"rewards/chosen": 0.348306604333826,
|
|
"rewards/margins": 0.27785791218600875,
|
|
"rewards/rejected": 0.07044869214781721,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 11.70887565612793,
|
|
"kl": 17.5533447265625,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35765926.55421687,
|
|
"logits/rejected": -37370281.55844156,
|
|
"logps/chosen": -475.8682228915663,
|
|
"logps/rejected": -385.58502435064935,
|
|
"loss": 0.5174,
|
|
"loss/base_kto": 3.7124946117401123,
|
|
"loss/total_with_kl": 4.139125347137451,
|
|
"metrics/advantage_var": 256.23419189453125,
|
|
"regularization/advantage_var": 256.23419189453125,
|
|
"regularization/kl": 0.4266298711299896,
|
|
"rewards/chosen": 0.38016680062535296,
|
|
"rewards/margins": 0.23725575659126205,
|
|
"rewards/rejected": 0.1429110440340909,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 20.91187858581543,
|
|
"kl": 15.539642333984375,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34957664.76555024,
|
|
"logits/rejected": -36433938.81776416,
|
|
"logps/chosen": -498.6603867623605,
|
|
"logps/rejected": -354.29424291730476,
|
|
"loss": 0.5111,
|
|
"loss/base_kto": 3.648345708847046,
|
|
"loss/total_with_kl": 4.088916301727295,
|
|
"metrics/advantage_var": 264.60699462890625,
|
|
"regularization/advantage_var": 264.60699462890625,
|
|
"regularization/kl": 0.4405706822872162,
|
|
"rewards/chosen": 0.3147894997726027,
|
|
"rewards/margins": 0.31643964897779736,
|
|
"rewards/rejected": -0.0016501492051946808,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 18.93696403503418,
|
|
"kl": 5.291294097900391,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36112220.16,
|
|
"logits/rejected": -37810000.9038168,
|
|
"logps/chosen": -495.9295,
|
|
"logps/rejected": -375.96839217557255,
|
|
"loss": 0.5246,
|
|
"loss/base_kto": 3.72355055809021,
|
|
"loss/total_with_kl": 4.196588039398193,
|
|
"metrics/advantage_var": 284.10650634765625,
|
|
"regularization/advantage_var": 284.10650634765625,
|
|
"regularization/kl": 0.47303733229637146,
|
|
"rewards/chosen": 0.058169354248046874,
|
|
"rewards/margins": 0.2757499486093303,
|
|
"rewards/rejected": -0.2175805943612834,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 16.306333541870117,
|
|
"kl": 7.787692546844482,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35247359.21230769,
|
|
"logits/rejected": -37101408.71111111,
|
|
"logps/chosen": -490.1022115384615,
|
|
"logps/rejected": -384.15761408730157,
|
|
"loss": 0.5195,
|
|
"loss/base_kto": 3.714895725250244,
|
|
"loss/total_with_kl": 4.1563310623168945,
|
|
"metrics/advantage_var": 265.1262512207031,
|
|
"regularization/advantage_var": 265.1262512207031,
|
|
"regularization/kl": 0.44143515825271606,
|
|
"rewards/chosen": 0.1767663339468149,
|
|
"rewards/margins": 0.26536319262233266,
|
|
"rewards/rejected": -0.08859685867551774,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 13.557397842407227,
|
|
"kl": 17.727249145507812,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35565049.67901234,
|
|
"logits/rejected": -39187663.39240506,
|
|
"logps/chosen": -491.52353395061726,
|
|
"logps/rejected": -355.3190268987342,
|
|
"loss": 0.5138,
|
|
"loss/base_kto": 3.6316330432891846,
|
|
"loss/total_with_kl": 4.11004114151001,
|
|
"metrics/advantage_var": 287.3321838378906,
|
|
"regularization/advantage_var": 287.3321838378906,
|
|
"regularization/kl": 0.4784080684185028,
|
|
"rewards/chosen": 0.3309371383101852,
|
|
"rewards/margins": 0.3535887140988521,
|
|
"rewards/rejected": -0.02265157578866693,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 15.566226959228516,
|
|
"kl": 9.367915153503418,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -34480345.31189711,
|
|
"logits/rejected": -36543945.5319149,
|
|
"logps/chosen": -478.0792805466238,
|
|
"logps/rejected": -374.4915463525836,
|
|
"loss": 0.5228,
|
|
"loss/base_kto": 3.7414352893829346,
|
|
"loss/total_with_kl": 4.18229866027832,
|
|
"metrics/advantage_var": 264.78271484375,
|
|
"regularization/advantage_var": 264.78271484375,
|
|
"regularization/kl": 0.4408632218837738,
|
|
"rewards/chosen": 0.10756603621209933,
|
|
"rewards/margins": 0.23945736218881025,
|
|
"rewards/rejected": -0.1318913259767109,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 15.71782112121582,
|
|
"kl": 14.42451000213623,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35432485.236363634,
|
|
"logits/rejected": -36481050.425806455,
|
|
"logps/chosen": -458.8940340909091,
|
|
"logps/rejected": -380.9310735887097,
|
|
"loss": 0.5155,
|
|
"loss/base_kto": 3.680269241333008,
|
|
"loss/total_with_kl": 4.124385356903076,
|
|
"metrics/advantage_var": 266.7364807128906,
|
|
"regularization/advantage_var": 266.7364807128906,
|
|
"regularization/kl": 0.4441162049770355,
|
|
"rewards/chosen": 0.32331404252485796,
|
|
"rewards/margins": 0.3041512369060796,
|
|
"rewards/rejected": 0.019162805618778353,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 13.159608840942383,
|
|
"kl": 8.973199844360352,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37341114.29497569,
|
|
"logits/rejected": -37675323.07692308,
|
|
"logps/chosen": -500.275425445705,
|
|
"logps/rejected": -365.4965592006033,
|
|
"loss": 0.5141,
|
|
"loss/base_kto": 3.6769776344299316,
|
|
"loss/total_with_kl": 4.112514495849609,
|
|
"metrics/advantage_var": 261.5839538574219,
|
|
"regularization/advantage_var": 261.5839538574219,
|
|
"regularization/kl": 0.4355372488498688,
|
|
"rewards/chosen": 0.18383294450212723,
|
|
"rewards/margins": 0.31317346009914127,
|
|
"rewards/rejected": -0.12934051559701404,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 13.037299156188965,
|
|
"kl": 9.277281761169434,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35099822.06431853,
|
|
"logits/rejected": -35899133.9585327,
|
|
"logps/chosen": -495.6490237366003,
|
|
"logps/rejected": -382.206115430622,
|
|
"loss": 0.5184,
|
|
"loss/base_kto": 3.6986634731292725,
|
|
"loss/total_with_kl": 4.147059917449951,
|
|
"metrics/advantage_var": 269.30712890625,
|
|
"regularization/advantage_var": 269.30712890625,
|
|
"regularization/kl": 0.44839635491371155,
|
|
"rewards/chosen": 0.12724044823171898,
|
|
"rewards/margins": 0.266642944228903,
|
|
"rewards/rejected": -0.13940249599718402,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 22.929607391357422,
|
|
"kl": 12.852584838867188,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36335391.219512194,
|
|
"logits/rejected": -37296974.76923077,
|
|
"logps/chosen": -477.5895579268293,
|
|
"logps/rejected": -366.32221554487177,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.6915135383605957,
|
|
"loss/total_with_kl": 4.127551078796387,
|
|
"metrics/advantage_var": 261.88427734375,
|
|
"regularization/advantage_var": 261.88427734375,
|
|
"regularization/kl": 0.4360373020172119,
|
|
"rewards/chosen": 0.23083547266518198,
|
|
"rewards/margins": 0.2442612217470137,
|
|
"rewards/rejected": -0.013425749081831712,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 16.30242919921875,
|
|
"kl": 22.94746208190918,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37376289.81132075,
|
|
"logits/rejected": -38749177.63975155,
|
|
"logps/chosen": -481.67845911949684,
|
|
"logps/rejected": -377.6274990295031,
|
|
"loss": 0.5149,
|
|
"loss/base_kto": 3.6707916259765625,
|
|
"loss/total_with_kl": 4.1191840171813965,
|
|
"metrics/advantage_var": 269.30450439453125,
|
|
"regularization/advantage_var": 269.30450439453125,
|
|
"regularization/kl": 0.44839200377464294,
|
|
"rewards/chosen": 0.37893259300375887,
|
|
"rewards/margins": 0.2631320908663911,
|
|
"rewards/rejected": 0.11580050213736777,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 14.475749969482422,
|
|
"kl": 16.794092178344727,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35113893.83647799,
|
|
"logits/rejected": -37618837.46583851,
|
|
"logps/chosen": -472.98432586477986,
|
|
"logps/rejected": -358.71797360248445,
|
|
"loss": 0.5175,
|
|
"loss/base_kto": 3.662067413330078,
|
|
"loss/total_with_kl": 4.140262603759766,
|
|
"metrics/advantage_var": 287.2043151855469,
|
|
"regularization/advantage_var": 287.2043151855469,
|
|
"regularization/kl": 0.4781951904296875,
|
|
"rewards/chosen": 0.32611971681222973,
|
|
"rewards/margins": 0.2916624952775481,
|
|
"rewards/rejected": 0.034457221534681615,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 16.73085594177246,
|
|
"kl": 20.94074821472168,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35394108.32628399,
|
|
"logits/rejected": -36212742.62783171,
|
|
"logps/chosen": -481.3646148036254,
|
|
"logps/rejected": -363.4192455501618,
|
|
"loss": 0.515,
|
|
"loss/base_kto": 3.6869637966156006,
|
|
"loss/total_with_kl": 4.119616508483887,
|
|
"metrics/advantage_var": 259.85150146484375,
|
|
"regularization/advantage_var": 259.85150146484375,
|
|
"regularization/kl": 0.43265271186828613,
|
|
"rewards/chosen": 0.384003515329966,
|
|
"rewards/margins": 0.24847656811095686,
|
|
"rewards/rejected": 0.13552694721900915,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 16.116498947143555,
|
|
"kl": 15.556981086730957,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36604658.526315786,
|
|
"logits/rejected": -37119865.139356814,
|
|
"logps/chosen": -481.2325558213716,
|
|
"logps/rejected": -371.94807618683,
|
|
"loss": 0.5156,
|
|
"loss/base_kto": 3.6460304260253906,
|
|
"loss/total_with_kl": 4.1249775886535645,
|
|
"metrics/advantage_var": 287.6559753417969,
|
|
"regularization/advantage_var": 287.6559753417969,
|
|
"regularization/kl": 0.47894716262817383,
|
|
"rewards/chosen": 0.2796679531748405,
|
|
"rewards/margins": 0.31520266437293165,
|
|
"rewards/rejected": -0.035534711198091144,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 13.209074020385742,
|
|
"kl": 18.143861770629883,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37562311.935587764,
|
|
"logits/rejected": -37459352.667678304,
|
|
"logps/chosen": -504.02480877616745,
|
|
"logps/rejected": -388.95549601669194,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.6370232105255127,
|
|
"loss/total_with_kl": 4.143251895904541,
|
|
"metrics/advantage_var": 304.0413513183594,
|
|
"regularization/advantage_var": 304.0413513183594,
|
|
"regularization/kl": 0.5062288641929626,
|
|
"rewards/chosen": 0.2887623859104519,
|
|
"rewards/margins": 0.31435846622160335,
|
|
"rewards/rejected": -0.025596080311151483,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 14.419571876525879,
|
|
"kl": 9.830085754394531,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35569604.73311897,
|
|
"logits/rejected": -36431735.051671736,
|
|
"logps/chosen": -479.87590434083603,
|
|
"logps/rejected": -383.4615311550152,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.7124288082122803,
|
|
"loss/total_with_kl": 4.144567012786865,
|
|
"metrics/advantage_var": 259.5424499511719,
|
|
"regularization/advantage_var": 259.5424499511719,
|
|
"regularization/kl": 0.4321381747722626,
|
|
"rewards/chosen": 0.10982839700876708,
|
|
"rewards/margins": 0.24909327771379175,
|
|
"rewards/rejected": -0.13926488070502469,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 20.648231506347656,
|
|
"kl": 14.493423461914062,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36495965.65853658,
|
|
"logits/rejected": -36950193.23076923,
|
|
"logps/chosen": -503.938262195122,
|
|
"logps/rejected": -382.3724959935897,
|
|
"loss": 0.5177,
|
|
"loss/base_kto": 3.6546566486358643,
|
|
"loss/total_with_kl": 4.1416168212890625,
|
|
"metrics/advantage_var": 292.46881103515625,
|
|
"regularization/advantage_var": 292.46881103515625,
|
|
"regularization/kl": 0.48696061968803406,
|
|
"rewards/chosen": 0.2888377119855183,
|
|
"rewards/margins": 0.31819413661062157,
|
|
"rewards/rejected": -0.02935642462510329,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 13.785265922546387,
|
|
"kl": 16.71992301940918,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35666898.34394904,
|
|
"logits/rejected": -35189175.754601225,
|
|
"logps/chosen": -469.50577229299364,
|
|
"logps/rejected": -365.37624616564415,
|
|
"loss": 0.5101,
|
|
"loss/base_kto": 3.6475751399993896,
|
|
"loss/total_with_kl": 4.080502510070801,
|
|
"metrics/advantage_var": 260.0164489746094,
|
|
"regularization/advantage_var": 260.0164489746094,
|
|
"regularization/kl": 0.43292737007141113,
|
|
"rewards/chosen": 0.2669396856028563,
|
|
"rewards/margins": 0.29887760072616976,
|
|
"rewards/rejected": -0.03193791512331348,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 43.316898345947266,
|
|
"kl": 20.487064361572266,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37338285.24471299,
|
|
"logits/rejected": -37374581.64401294,
|
|
"logps/chosen": -475.091625755287,
|
|
"logps/rejected": -369.8734324433657,
|
|
"loss": 0.5204,
|
|
"loss/base_kto": 3.684366226196289,
|
|
"loss/total_with_kl": 4.16303014755249,
|
|
"metrics/advantage_var": 287.4859313964844,
|
|
"regularization/advantage_var": 287.4859313964844,
|
|
"regularization/kl": 0.4786640703678131,
|
|
"rewards/chosen": 0.349972796944333,
|
|
"rewards/margins": 0.27057078691244457,
|
|
"rewards/rejected": 0.0794020100318884,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 21.38011932373047,
|
|
"kl": 24.615724563598633,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35794931.1799687,
|
|
"logits/rejected": -35510048.34945398,
|
|
"logps/chosen": -491.32707355242565,
|
|
"logps/rejected": -372.66712168486737,
|
|
"loss": 0.5117,
|
|
"loss/base_kto": 3.600149631500244,
|
|
"loss/total_with_kl": 4.0938029289245605,
|
|
"metrics/advantage_var": 296.4882507324219,
|
|
"regularization/advantage_var": 296.4882507324219,
|
|
"regularization/kl": 0.493652880191803,
|
|
"rewards/chosen": 0.43623239818686427,
|
|
"rewards/margins": 0.3389319683889106,
|
|
"rewards/rejected": 0.09730042979795364,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 145.93531799316406,
|
|
"kl": 19.115631103515625,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36294998.420382164,
|
|
"logits/rejected": -37305601.57055215,
|
|
"logps/chosen": -474.3715167197452,
|
|
"logps/rejected": -359.26993865030676,
|
|
"loss": 0.5191,
|
|
"loss/base_kto": 3.640221357345581,
|
|
"loss/total_with_kl": 4.152987480163574,
|
|
"metrics/advantage_var": 307.96746826171875,
|
|
"regularization/advantage_var": 307.96746826171875,
|
|
"regularization/kl": 0.5127658247947693,
|
|
"rewards/chosen": 0.33320646832703027,
|
|
"rewards/margins": 0.32371776870069274,
|
|
"rewards/rejected": 0.009488699626337532,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 16.139719009399414,
|
|
"kl": 15.433197021484375,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35393484.8798752,
|
|
"logits/rejected": -36672468.73239437,
|
|
"logps/chosen": -507.7396158346334,
|
|
"logps/rejected": -372.2236404538341,
|
|
"loss": 0.5206,
|
|
"loss/base_kto": 3.639004945755005,
|
|
"loss/total_with_kl": 4.164422035217285,
|
|
"metrics/advantage_var": 315.56585693359375,
|
|
"regularization/advantage_var": 315.56585693359375,
|
|
"regularization/kl": 0.525417149066925,
|
|
"rewards/chosen": 0.20587270085041684,
|
|
"rewards/margins": 0.3296515679003728,
|
|
"rewards/rejected": -0.12377886704995598,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 31.113555908203125,
|
|
"kl": 15.1028470993042,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36330912.94851794,
|
|
"logits/rejected": -37367312.87912088,
|
|
"logps/chosen": -487.8967433697348,
|
|
"logps/rejected": -362.51091542386183,
|
|
"loss": 0.5182,
|
|
"loss/base_kto": 3.4688007831573486,
|
|
"loss/total_with_kl": 4.145680904388428,
|
|
"metrics/advantage_var": 406.5345153808594,
|
|
"regularization/advantage_var": 406.5345153808594,
|
|
"regularization/kl": 0.6768800020217896,
|
|
"rewards/chosen": 0.35281691052798364,
|
|
"rewards/margins": 0.5150296476548356,
|
|
"rewards/rejected": -0.16221273712685194,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 21.974807739257812,
|
|
"kl": 21.096019744873047,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -34902750.12923077,
|
|
"logits/rejected": -35730493.765079364,
|
|
"logps/chosen": -483.70221153846154,
|
|
"logps/rejected": -382.71502976190476,
|
|
"loss": 0.5228,
|
|
"loss/base_kto": 3.095506429672241,
|
|
"loss/total_with_kl": 4.1820855140686035,
|
|
"metrics/advantage_var": 652.60009765625,
|
|
"regularization/advantage_var": 652.60009765625,
|
|
"regularization/kl": 1.0865792036056519,
|
|
"rewards/chosen": 0.7133823805588942,
|
|
"rewards/margins": 0.9873754458025699,
|
|
"rewards/rejected": -0.2739930652436756,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 11.445603370666504,
|
|
"kl": 24.443540573120117,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34818446.222222224,
|
|
"logits/rejected": -36179767.088607594,
|
|
"logps/chosen": -493.6755401234568,
|
|
"logps/rejected": -398.9047913370253,
|
|
"loss": 0.5363,
|
|
"loss/base_kto": 3.096897602081299,
|
|
"loss/total_with_kl": 4.290025234222412,
|
|
"metrics/advantage_var": 716.59326171875,
|
|
"regularization/advantage_var": 716.59326171875,
|
|
"regularization/kl": 1.1931277513504028,
|
|
"rewards/chosen": 0.7552757734133874,
|
|
"rewards/margins": 0.9890814177746661,
|
|
"rewards/rejected": -0.23380564436127868,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 20.024269104003906,
|
|
"kl": 26.2730712890625,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -34908460.1095008,
|
|
"logits/rejected": -35962824.060698025,
|
|
"logps/chosen": -488.4697564412238,
|
|
"logps/rejected": -393.1842754172989,
|
|
"loss": 0.5048,
|
|
"loss/base_kto": 3.1360538005828857,
|
|
"loss/total_with_kl": 4.038689613342285,
|
|
"metrics/advantage_var": 542.1234741210938,
|
|
"regularization/advantage_var": 542.1234741210938,
|
|
"regularization/kl": 0.9026355743408203,
|
|
"rewards/chosen": 0.757409530369364,
|
|
"rewards/margins": 0.9136508242924481,
|
|
"rewards/rejected": -0.15624129392308422,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 21.529926300048828,
|
|
"kl": 9.573410987854004,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34975737.4464,
|
|
"logits/rejected": -36086504.15877862,
|
|
"logps/chosen": -486.6575,
|
|
"logps/rejected": -389.136641221374,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.2002956867218018,
|
|
"loss/total_with_kl": 4.195446968078613,
|
|
"metrics/advantage_var": 597.6884765625,
|
|
"regularization/advantage_var": 597.6884765625,
|
|
"regularization/kl": 0.9951512217521667,
|
|
"rewards/chosen": 0.388057763671875,
|
|
"rewards/margins": 0.8503276937470181,
|
|
"rewards/rejected": -0.4622699300751431,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 14.2310791015625,
|
|
"kl": 16.73166847229004,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35703589.717868336,
|
|
"logits/rejected": -36457500.71028037,
|
|
"logps/chosen": -468.9491085423198,
|
|
"logps/rejected": -361.90678543613706,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.1490395069122314,
|
|
"loss/total_with_kl": 4.14325475692749,
|
|
"metrics/advantage_var": 597.1264038085938,
|
|
"regularization/advantage_var": 597.1264038085938,
|
|
"regularization/kl": 0.9942154288291931,
|
|
"rewards/chosen": 0.6469504138145327,
|
|
"rewards/margins": 0.93025283978136,
|
|
"rewards/rejected": -0.2833024259668273,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 19.249755859375,
|
|
"kl": 10.497282981872559,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35211503.45846154,
|
|
"logits/rejected": -36472604.44444445,
|
|
"logps/chosen": -474.0904807692308,
|
|
"logps/rejected": -384.2449404761905,
|
|
"loss": 0.5222,
|
|
"loss/base_kto": 3.2282378673553467,
|
|
"loss/total_with_kl": 4.177388668060303,
|
|
"metrics/advantage_var": 570.0606689453125,
|
|
"regularization/advantage_var": 570.0606689453125,
|
|
"regularization/kl": 0.9491510391235352,
|
|
"rewards/chosen": 0.4077448918269231,
|
|
"rewards/margins": 0.8156021025009061,
|
|
"rewards/rejected": -0.4078572106739831,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 12.673859596252441,
|
|
"kl": 22.763656616210938,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35433958.04608295,
|
|
"logits/rejected": -36659326.98251192,
|
|
"logps/chosen": -486.46409370199694,
|
|
"logps/rejected": -354.6059966216216,
|
|
"loss": 0.5003,
|
|
"loss/base_kto": 3.0805881023406982,
|
|
"loss/total_with_kl": 4.0026164054870605,
|
|
"metrics/advantage_var": 553.770751953125,
|
|
"regularization/advantage_var": 553.770751953125,
|
|
"regularization/kl": 0.9220282435417175,
|
|
"rewards/chosen": 0.7849215318530386,
|
|
"rewards/margins": 0.9836626866071436,
|
|
"rewards/rejected": -0.19874115475410498,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 17.543062210083008,
|
|
"kl": 18.01152992248535,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34316418.238473766,
|
|
"logits/rejected": -35322412.82949309,
|
|
"logps/chosen": -472.60741255961847,
|
|
"logps/rejected": -347.34483006912444,
|
|
"loss": 0.5108,
|
|
"loss/base_kto": 3.2038142681121826,
|
|
"loss/total_with_kl": 4.086097717285156,
|
|
"metrics/advantage_var": 529.9000244140625,
|
|
"regularization/advantage_var": 529.9000244140625,
|
|
"regularization/kl": 0.8822835087776184,
|
|
"rewards/chosen": 0.5735934064953051,
|
|
"rewards/margins": 0.8595392226525391,
|
|
"rewards/rejected": -0.2859458161572341,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 9.224899291992188,
|
|
"kl": 18.6295223236084,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34802521.6,
|
|
"logits/rejected": -36403760.0,
|
|
"logps/chosen": -433.265234375,
|
|
"logps/rejected": -375.4929443359375,
|
|
"loss": 0.5261,
|
|
"loss/base_kto": 3.145615339279175,
|
|
"loss/total_with_kl": 4.209061622619629,
|
|
"metrics/advantage_var": 638.7062377929688,
|
|
"regularization/advantage_var": 638.7062377929688,
|
|
"regularization/kl": 1.063446044921875,
|
|
"rewards/chosen": 0.6340777397155761,
|
|
"rewards/margins": 0.9245184659957886,
|
|
"rewards/rejected": -0.2904407262802124,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 16.165605545043945,
|
|
"kl": 17.752416610717773,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -33469711.203852326,
|
|
"logits/rejected": -36636893.32115677,
|
|
"logps/chosen": -511.0252307383628,
|
|
"logps/rejected": -389.2472650304414,
|
|
"loss": 0.5349,
|
|
"loss/base_kto": 3.1670024394989014,
|
|
"loss/total_with_kl": 4.279271125793457,
|
|
"metrics/advantage_var": 668.0293579101562,
|
|
"regularization/advantage_var": 668.0293579101562,
|
|
"regularization/kl": 1.1122688055038452,
|
|
"rewards/chosen": 0.6031466709094101,
|
|
"rewards/margins": 0.9128984166820985,
|
|
"rewards/rejected": -0.3097517457726884,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 119.1275405883789,
|
|
"kl": 16.88669204711914,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -36045253.67088608,
|
|
"logits/rejected": -34577449.086419754,
|
|
"logps/chosen": -467.2184038765823,
|
|
"logps/rejected": -375.0442708333333,
|
|
"loss": 0.665,
|
|
"loss/base_kto": 3.161041021347046,
|
|
"loss/total_with_kl": 5.320053577423096,
|
|
"metrics/advantage_var": 1296.7041015625,
|
|
"regularization/advantage_var": 1296.7041015625,
|
|
"regularization/kl": 2.15901255607605,
|
|
"rewards/chosen": 0.6263412282436709,
|
|
"rewards/margins": 0.9371888756398057,
|
|
"rewards/rejected": -0.31084764739613474,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 20.290742874145508,
|
|
"kl": 12.173848152160645,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34316786.832072616,
|
|
"logits/rejected": -36752296.323101774,
|
|
"logps/chosen": -471.86497730711045,
|
|
"logps/rejected": -377.8185581583199,
|
|
"loss": 0.5494,
|
|
"loss/base_kto": 3.226526975631714,
|
|
"loss/total_with_kl": 4.395519256591797,
|
|
"metrics/advantage_var": 702.097412109375,
|
|
"regularization/advantage_var": 702.097412109375,
|
|
"regularization/kl": 1.1689921617507935,
|
|
"rewards/chosen": 0.5521525095883368,
|
|
"rewards/margins": 0.8078719178608782,
|
|
"rewards/rejected": -0.2557194082725414,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 17.557085037231445,
|
|
"kl": 25.647724151611328,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34383122.583703704,
|
|
"logits/rejected": -36638151.299173556,
|
|
"logps/chosen": -481.93148148148146,
|
|
"logps/rejected": -360.85795454545456,
|
|
"loss": 0.5108,
|
|
"loss/base_kto": 3.1940364837646484,
|
|
"loss/total_with_kl": 4.08669900894165,
|
|
"metrics/advantage_var": 536.1336059570312,
|
|
"regularization/advantage_var": 536.1336059570312,
|
|
"regularization/kl": 0.8926624655723572,
|
|
"rewards/chosen": 0.7570982982494213,
|
|
"rewards/margins": 0.8593505459573242,
|
|
"rewards/rejected": -0.1022522477079029,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 12.21741771697998,
|
|
"kl": 22.415632247924805,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35247341.742033385,
|
|
"logits/rejected": -36756554.20289855,
|
|
"logps/chosen": -483.63002655538696,
|
|
"logps/rejected": -381.67507045088564,
|
|
"loss": 0.5783,
|
|
"loss/base_kto": 3.184499502182007,
|
|
"loss/total_with_kl": 4.626182556152344,
|
|
"metrics/advantage_var": 865.8757934570312,
|
|
"regularization/advantage_var": 865.8757934570312,
|
|
"regularization/kl": 1.4416831731796265,
|
|
"rewards/chosen": 0.7054853656404116,
|
|
"rewards/margins": 0.8441251013552366,
|
|
"rewards/rejected": -0.13863973571482488,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 9.0669527053833,
|
|
"kl": 17.679346084594727,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35166431.3609831,
|
|
"logits/rejected": -37391400.699523054,
|
|
"logps/chosen": -497.14170506912444,
|
|
"logps/rejected": -367.95933525437204,
|
|
"loss": 0.5164,
|
|
"loss/base_kto": 3.18158221244812,
|
|
"loss/total_with_kl": 4.131343364715576,
|
|
"metrics/advantage_var": 570.4268798828125,
|
|
"regularization/advantage_var": 570.4268798828125,
|
|
"regularization/kl": 0.9497607350349426,
|
|
"rewards/chosen": 0.6133537233882969,
|
|
"rewards/margins": 0.8766409022405177,
|
|
"rewards/rejected": -0.2632871788522208,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 19.357873916625977,
|
|
"kl": 20.663843154907227,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35012993.58024691,
|
|
"logits/rejected": -36169468.75949367,
|
|
"logps/chosen": -477.03211805555554,
|
|
"logps/rejected": -357.1598595727848,
|
|
"loss": 0.502,
|
|
"loss/base_kto": 3.1430957317352295,
|
|
"loss/total_with_kl": 4.015756607055664,
|
|
"metrics/advantage_var": 524.1205444335938,
|
|
"regularization/advantage_var": 524.1205444335938,
|
|
"regularization/kl": 0.872660756111145,
|
|
"rewards/chosen": 0.6914439260223766,
|
|
"rewards/margins": 0.8968758038346294,
|
|
"rewards/rejected": -0.20543187781225278,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 14.7743558883667,
|
|
"kl": 22.073400497436523,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35463370.54945055,
|
|
"logits/rejected": -36603024.92068429,
|
|
"logps/chosen": -469.2876766091052,
|
|
"logps/rejected": -369.5140211897356,
|
|
"loss": 0.5372,
|
|
"loss/base_kto": 3.152519941329956,
|
|
"loss/total_with_kl": 4.297989368438721,
|
|
"metrics/advantage_var": 687.9698486328125,
|
|
"regularization/advantage_var": 687.9698486328125,
|
|
"regularization/kl": 1.1454696655273438,
|
|
"rewards/chosen": 0.6753101199163559,
|
|
"rewards/margins": 0.8877968022920606,
|
|
"rewards/rejected": -0.2124866823757047,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 25.720378875732422,
|
|
"kl": 18.7586669921875,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34098366.69796557,
|
|
"logits/rejected": -36353859.49453978,
|
|
"logps/chosen": -496.879010172144,
|
|
"logps/rejected": -393.6916195397816,
|
|
"loss": 0.5312,
|
|
"loss/base_kto": 3.1742589473724365,
|
|
"loss/total_with_kl": 4.249287128448486,
|
|
"metrics/advantage_var": 645.6625366210938,
|
|
"regularization/advantage_var": 645.6625366210938,
|
|
"regularization/kl": 1.0750280618667603,
|
|
"rewards/chosen": 0.6056689051954959,
|
|
"rewards/margins": 0.8555184359198889,
|
|
"rewards/rejected": -0.24984953072439303,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 13.962899208068848,
|
|
"kl": 23.922412872314453,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -32700181.85365854,
|
|
"logits/rejected": -36528456.20512821,
|
|
"logps/chosen": -473.018340320122,
|
|
"logps/rejected": -367.9064503205128,
|
|
"loss": 0.5293,
|
|
"loss/base_kto": 3.193666934967041,
|
|
"loss/total_with_kl": 4.23423957824707,
|
|
"metrics/advantage_var": 624.9685668945312,
|
|
"regularization/advantage_var": 624.9685668945312,
|
|
"regularization/kl": 1.0405725240707397,
|
|
"rewards/chosen": 0.7202913237781059,
|
|
"rewards/margins": 0.872715549218498,
|
|
"rewards/rejected": -0.15242422544039214,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 34.06401062011719,
|
|
"kl": 27.436681747436523,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35291384.8224299,
|
|
"logits/rejected": -35755377.153605014,
|
|
"logps/chosen": -462.1151674454829,
|
|
"logps/rejected": -369.5713166144201,
|
|
"loss": 0.5394,
|
|
"loss/base_kto": 3.1750638484954834,
|
|
"loss/total_with_kl": 4.315387725830078,
|
|
"metrics/advantage_var": 684.8792114257812,
|
|
"regularization/advantage_var": 684.8792114257812,
|
|
"regularization/kl": 1.1403238773345947,
|
|
"rewards/chosen": 0.713312642225224,
|
|
"rewards/margins": 0.8410065120760762,
|
|
"rewards/rejected": -0.12769386985085227,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 18.86966896057129,
|
|
"kl": 15.892166137695312,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35464024.376012966,
|
|
"logits/rejected": -36400058.49773756,
|
|
"logps/chosen": -470.35929902755265,
|
|
"logps/rejected": -350.68240950226243,
|
|
"loss": 0.512,
|
|
"loss/base_kto": 3.220384359359741,
|
|
"loss/total_with_kl": 4.095946311950684,
|
|
"metrics/advantage_var": 525.8631591796875,
|
|
"regularization/advantage_var": 525.8631591796875,
|
|
"regularization/kl": 0.8755621314048767,
|
|
"rewards/chosen": 0.6322690200187399,
|
|
"rewards/margins": 0.8324806499869831,
|
|
"rewards/rejected": -0.2002116299682433,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 27.72390365600586,
|
|
"kl": 16.47442054748535,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35167950.42539682,
|
|
"logits/rejected": -36979809.673846155,
|
|
"logps/chosen": -497.8609623015873,
|
|
"logps/rejected": -385.4046153846154,
|
|
"loss": 0.5057,
|
|
"loss/base_kto": 3.1687557697296143,
|
|
"loss/total_with_kl": 4.045971870422363,
|
|
"metrics/advantage_var": 526.8564453125,
|
|
"regularization/advantage_var": 526.8564453125,
|
|
"regularization/kl": 0.8772159814834595,
|
|
"rewards/chosen": 0.5816710941375248,
|
|
"rewards/margins": 0.8957860843718999,
|
|
"rewards/rejected": -0.314114990234375,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 13.024700164794922,
|
|
"kl": 14.6386079788208,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35346368.41830065,
|
|
"logits/rejected": -36299034.05988024,
|
|
"logps/chosen": -485.52359068627453,
|
|
"logps/rejected": -395.9764221556886,
|
|
"loss": 0.5054,
|
|
"loss/base_kto": 3.2019081115722656,
|
|
"loss/total_with_kl": 4.043558597564697,
|
|
"metrics/advantage_var": 505.49566650390625,
|
|
"regularization/advantage_var": 505.49566650390625,
|
|
"regularization/kl": 0.8416501879692078,
|
|
"rewards/chosen": 0.4972510244332108,
|
|
"rewards/margins": 0.8567804854087206,
|
|
"rewards/rejected": -0.3595294609755099,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 15.260805130004883,
|
|
"kl": 17.375057220458984,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34435508.56426332,
|
|
"logits/rejected": -35684664.62305296,
|
|
"logps/chosen": -495.2782621473354,
|
|
"logps/rejected": -374.00642523364485,
|
|
"loss": 0.5491,
|
|
"loss/base_kto": 3.1951146125793457,
|
|
"loss/total_with_kl": 4.392637729644775,
|
|
"metrics/advantage_var": 719.2330322265625,
|
|
"regularization/advantage_var": 719.2330322265625,
|
|
"regularization/kl": 1.1975229978561401,
|
|
"rewards/chosen": 0.6216823852921728,
|
|
"rewards/margins": 0.816191802389318,
|
|
"rewards/rejected": -0.19450941709714514,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 13.97712230682373,
|
|
"kl": 10.879806518554688,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34823172.82260597,
|
|
"logits/rejected": -36052233.953343704,
|
|
"logps/chosen": -498.48302590266877,
|
|
"logps/rejected": -372.73617321150857,
|
|
"loss": 0.5146,
|
|
"loss/base_kto": 3.2071304321289062,
|
|
"loss/total_with_kl": 4.116452217102051,
|
|
"metrics/advantage_var": 546.1390991210938,
|
|
"regularization/advantage_var": 546.1390991210938,
|
|
"regularization/kl": 0.9093216061592102,
|
|
"rewards/chosen": 0.4803002086501668,
|
|
"rewards/margins": 0.8660376737566622,
|
|
"rewards/rejected": -0.3857374651064954,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 22.177412033081055,
|
|
"kl": 15.012435913085938,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -37029306.469135806,
|
|
"logits/rejected": -36475217.01265823,
|
|
"logps/chosen": -480.08256172839504,
|
|
"logps/rejected": -379.47804588607596,
|
|
"loss": 0.5276,
|
|
"loss/base_kto": 3.254469394683838,
|
|
"loss/total_with_kl": 4.220569133758545,
|
|
"metrics/advantage_var": 580.240234375,
|
|
"regularization/advantage_var": 580.240234375,
|
|
"regularization/kl": 0.9660999178886414,
|
|
"rewards/chosen": 0.5825492482126495,
|
|
"rewards/margins": 0.7777795149285563,
|
|
"rewards/rejected": -0.19523026671590685,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 23.869699478149414,
|
|
"kl": 12.002751350402832,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34679942.4,
|
|
"logits/rejected": -35694416.0,
|
|
"logps/chosen": -508.879736328125,
|
|
"logps/rejected": -363.1947265625,
|
|
"loss": 0.5073,
|
|
"loss/base_kto": 3.2425103187561035,
|
|
"loss/total_with_kl": 4.058449745178223,
|
|
"metrics/advantage_var": 490.0536193847656,
|
|
"regularization/advantage_var": 490.0536193847656,
|
|
"regularization/kl": 0.8159392476081848,
|
|
"rewards/chosen": 0.46154537200927737,
|
|
"rewards/margins": 0.7988351821899414,
|
|
"rewards/rejected": -0.33728981018066406,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 22.328872680664062,
|
|
"kl": 11.328975677490234,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33229568.41761827,
|
|
"logits/rejected": -34615025.79910045,
|
|
"logps/chosen": -463.5056076672104,
|
|
"logps/rejected": -372.407561844078,
|
|
"loss": 0.5118,
|
|
"loss/base_kto": 3.231456756591797,
|
|
"loss/total_with_kl": 4.094671249389648,
|
|
"metrics/advantage_var": 518.447265625,
|
|
"regularization/advantage_var": 518.447265625,
|
|
"regularization/kl": 0.8632146716117859,
|
|
"rewards/chosen": 0.41919343584318924,
|
|
"rewards/margins": 0.8015186840467242,
|
|
"rewards/rejected": -0.382325248203535,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 17.927732467651367,
|
|
"kl": 13.05345630645752,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34053565.217391305,
|
|
"logits/rejected": -35702049.81132075,
|
|
"logps/chosen": -461.24844720496895,
|
|
"logps/rejected": -407.92162932389937,
|
|
"loss": 0.5019,
|
|
"loss/base_kto": 3.2033932209014893,
|
|
"loss/total_with_kl": 4.015069484710693,
|
|
"metrics/advantage_var": 487.4932556152344,
|
|
"regularization/advantage_var": 487.4932556152344,
|
|
"regularization/kl": 0.8116763234138489,
|
|
"rewards/chosen": 0.5069902313421972,
|
|
"rewards/margins": 0.8418340408418779,
|
|
"rewards/rejected": -0.3348438094996806,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 20.6157283782959,
|
|
"kl": 14.827483177185059,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34930153.73913044,
|
|
"logits/rejected": -36487099.629742034,
|
|
"logps/chosen": -484.662037037037,
|
|
"logps/rejected": -393.36428300455236,
|
|
"loss": 0.5146,
|
|
"loss/base_kto": 3.2425010204315186,
|
|
"loss/total_with_kl": 4.116556644439697,
|
|
"metrics/advantage_var": 524.95849609375,
|
|
"regularization/advantage_var": 524.95849609375,
|
|
"regularization/kl": 0.8740558624267578,
|
|
"rewards/chosen": 0.4873495547283867,
|
|
"rewards/margins": 0.8078777385221912,
|
|
"rewards/rejected": -0.32052818379380454,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 12.403368949890137,
|
|
"kl": 10.325544357299805,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33934097.22691706,
|
|
"logits/rejected": -34286149.49141966,
|
|
"logps/chosen": -465.74623435054775,
|
|
"logps/rejected": -363.0441205148206,
|
|
"loss": 0.5182,
|
|
"loss/base_kto": 3.248669147491455,
|
|
"loss/total_with_kl": 4.1453728675842285,
|
|
"metrics/advantage_var": 538.5609741210938,
|
|
"regularization/advantage_var": 538.5609741210938,
|
|
"regularization/kl": 0.8967038989067078,
|
|
"rewards/chosen": 0.4542280743379548,
|
|
"rewards/margins": 0.8177103785227003,
|
|
"rewards/rejected": -0.36348230418474553,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 16.25936508178711,
|
|
"kl": 12.982745170593262,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33951469.83225807,
|
|
"logits/rejected": -35572503.27272727,
|
|
"logps/chosen": -485.821875,
|
|
"logps/rejected": -373.7669270833333,
|
|
"loss": 0.5075,
|
|
"loss/base_kto": 3.1363184452056885,
|
|
"loss/total_with_kl": 4.059669017791748,
|
|
"metrics/advantage_var": 554.5648803710938,
|
|
"regularization/advantage_var": 554.5648803710938,
|
|
"regularization/kl": 0.9233505129814148,
|
|
"rewards/chosen": 0.539885490171371,
|
|
"rewards/margins": 0.9372862710514609,
|
|
"rewards/rejected": -0.39740078088008995,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 16.2618408203125,
|
|
"kl": 15.021658897399902,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35530640.96385542,
|
|
"logits/rejected": -34702435.74025974,
|
|
"logps/chosen": -486.8272307981928,
|
|
"logps/rejected": -384.74543425324674,
|
|
"loss": 0.508,
|
|
"loss/base_kto": 3.16685152053833,
|
|
"loss/total_with_kl": 4.064095497131348,
|
|
"metrics/advantage_var": 538.8851318359375,
|
|
"regularization/advantage_var": 538.8851318359375,
|
|
"regularization/kl": 0.8972436785697937,
|
|
"rewards/chosen": 0.6191224247576242,
|
|
"rewards/margins": 0.8982581907250553,
|
|
"rewards/rejected": -0.279135765967431,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 10.686800956726074,
|
|
"kl": 13.021166801452637,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34040484.256651014,
|
|
"logits/rejected": -35551113.78471139,
|
|
"logps/chosen": -484.4517312206573,
|
|
"logps/rejected": -367.4321129095164,
|
|
"loss": 0.4938,
|
|
"loss/base_kto": 3.207645893096924,
|
|
"loss/total_with_kl": 3.9501023292541504,
|
|
"metrics/advantage_var": 445.91973876953125,
|
|
"regularization/advantage_var": 445.91973876953125,
|
|
"regularization/kl": 0.7424563765525818,
|
|
"rewards/chosen": 0.5291195005318369,
|
|
"rewards/margins": 0.8354601867764742,
|
|
"rewards/rejected": -0.3063406862446373,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 16.483449935913086,
|
|
"kl": 14.305689811706543,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35764896.29179332,
|
|
"logits/rejected": -35511905.1318328,
|
|
"logps/chosen": -478.85011398176295,
|
|
"logps/rejected": -377.6265072347267,
|
|
"loss": 0.5013,
|
|
"loss/base_kto": 3.1524693965911865,
|
|
"loss/total_with_kl": 4.010059356689453,
|
|
"metrics/advantage_var": 515.0692138671875,
|
|
"regularization/advantage_var": 515.0692138671875,
|
|
"regularization/kl": 0.8575901985168457,
|
|
"rewards/chosen": 0.5934567900776501,
|
|
"rewards/margins": 0.9168128052002384,
|
|
"rewards/rejected": -0.32335601512258844,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 13.733931541442871,
|
|
"kl": 15.422091484069824,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35332454.74829932,
|
|
"logits/rejected": -36086943.8150289,
|
|
"logps/chosen": -464.26589073129253,
|
|
"logps/rejected": -377.4020953757225,
|
|
"loss": 0.4931,
|
|
"loss/base_kto": 3.217292070388794,
|
|
"loss/total_with_kl": 3.944584608078003,
|
|
"metrics/advantage_var": 436.8122253417969,
|
|
"regularization/advantage_var": 436.8122253417969,
|
|
"regularization/kl": 0.7272922396659851,
|
|
"rewards/chosen": 0.48512465288849915,
|
|
"rewards/margins": 0.7993540778667664,
|
|
"rewards/rejected": -0.3142294249782672,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 18.97408103942871,
|
|
"kl": 11.096129417419434,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34857671.80487805,
|
|
"logits/rejected": -36135020.307692304,
|
|
"logps/chosen": -495.4438833841463,
|
|
"logps/rejected": -358.8788060897436,
|
|
"loss": 0.5214,
|
|
"loss/base_kto": 3.1606152057647705,
|
|
"loss/total_with_kl": 4.170856952667236,
|
|
"metrics/advantage_var": 606.7518920898438,
|
|
"regularization/advantage_var": 606.7518920898438,
|
|
"regularization/kl": 1.0102418661117554,
|
|
"rewards/chosen": 0.5045560976354088,
|
|
"rewards/margins": 0.9385915700162181,
|
|
"rewards/rejected": -0.4340354723808093,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 26.482131958007812,
|
|
"kl": 12.319792747497559,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34331154.17751479,
|
|
"logits/rejected": -36180422.357615896,
|
|
"logps/chosen": -484.29391642011836,
|
|
"logps/rejected": -370.2840438741722,
|
|
"loss": 0.5038,
|
|
"loss/base_kto": 3.2293007373809814,
|
|
"loss/total_with_kl": 4.0306596755981445,
|
|
"metrics/advantage_var": 481.29638671875,
|
|
"regularization/advantage_var": 481.29638671875,
|
|
"regularization/kl": 0.8013584017753601,
|
|
"rewards/chosen": 0.568287414911936,
|
|
"rewards/margins": 0.8108820517406301,
|
|
"rewards/rejected": -0.24259463682869412,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 7.906752586364746,
|
|
"kl": 14.77958869934082,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34494963.26368159,
|
|
"logits/rejected": -36518837.66518518,
|
|
"logps/chosen": -481.1639718076285,
|
|
"logps/rejected": -360.60342592592593,
|
|
"loss": 0.4732,
|
|
"loss/base_kto": 3.1612300872802734,
|
|
"loss/total_with_kl": 3.785780429840088,
|
|
"metrics/advantage_var": 375.10516357421875,
|
|
"regularization/advantage_var": 375.10516357421875,
|
|
"regularization/kl": 0.6245501041412354,
|
|
"rewards/chosen": 0.5704252581493574,
|
|
"rewards/margins": 0.8595004625041027,
|
|
"rewards/rejected": -0.2890752043547454,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 7.699523448944092,
|
|
"kl": 7.288259983062744,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34509809.15942029,
|
|
"logits/rejected": -35382292.20030349,
|
|
"logps/chosen": -465.41183574879227,
|
|
"logps/rejected": -375.4367886949924,
|
|
"loss": 0.4662,
|
|
"loss/base_kto": 3.136625051498413,
|
|
"loss/total_with_kl": 3.729235887527466,
|
|
"metrics/advantage_var": 355.9222106933594,
|
|
"regularization/advantage_var": 355.9222106933594,
|
|
"regularization/kl": 0.5926105380058289,
|
|
"rewards/chosen": 0.4267546673711755,
|
|
"rewards/margins": 0.8816557533709146,
|
|
"rewards/rejected": -0.45490108599973916,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 11.611719131469727,
|
|
"kl": 9.821981430053711,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33172362.598726116,
|
|
"logits/rejected": -35869447.85276074,
|
|
"logps/chosen": -457.77846337579615,
|
|
"logps/rejected": -379.2510784125767,
|
|
"loss": 0.4636,
|
|
"loss/base_kto": 3.097532272338867,
|
|
"loss/total_with_kl": 3.708944320678711,
|
|
"metrics/advantage_var": 367.21441650390625,
|
|
"regularization/advantage_var": 367.21441650390625,
|
|
"regularization/kl": 0.611411988735199,
|
|
"rewards/chosen": 0.5261565287401722,
|
|
"rewards/margins": 0.9095836396579013,
|
|
"rewards/rejected": -0.38342711091772913,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 10.034419059753418,
|
|
"kl": 13.82537841796875,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34067506.7244582,
|
|
"logits/rejected": -36231445.8044164,
|
|
"logps/chosen": -482.71579914860683,
|
|
"logps/rejected": -377.22473876182966,
|
|
"loss": 0.4638,
|
|
"loss/base_kto": 3.110532760620117,
|
|
"loss/total_with_kl": 3.7102158069610596,
|
|
"metrics/advantage_var": 360.1698913574219,
|
|
"regularization/advantage_var": 360.1698913574219,
|
|
"regularization/kl": 0.5996828675270081,
|
|
"rewards/chosen": 0.5588428509124661,
|
|
"rewards/margins": 0.9019656521065217,
|
|
"rewards/rejected": -0.3431228011940556,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 9.551857948303223,
|
|
"kl": 13.70224666595459,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -32953689.854304634,
|
|
"logits/rejected": -34677993.27810651,
|
|
"logps/chosen": -450.9413286423841,
|
|
"logps/rejected": -373.42111224112426,
|
|
"loss": 0.4644,
|
|
"loss/base_kto": 3.104682207107544,
|
|
"loss/total_with_kl": 3.7153890132904053,
|
|
"metrics/advantage_var": 366.7907409667969,
|
|
"regularization/advantage_var": 366.7907409667969,
|
|
"regularization/kl": 0.610706627368927,
|
|
"rewards/chosen": 0.5498176726284406,
|
|
"rewards/margins": 0.910659051286216,
|
|
"rewards/rejected": -0.3608413786577755,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 26.569332122802734,
|
|
"kl": 11.5601224899292,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34094174.1958457,
|
|
"logits/rejected": -35801260.35643564,
|
|
"logps/chosen": -467.55313427299706,
|
|
"logps/rejected": -371.6885571369637,
|
|
"loss": 0.477,
|
|
"loss/base_kto": 3.153583526611328,
|
|
"loss/total_with_kl": 3.815898895263672,
|
|
"metrics/advantage_var": 397.7869567871094,
|
|
"regularization/advantage_var": 397.7869567871094,
|
|
"regularization/kl": 0.662315309047699,
|
|
"rewards/chosen": 0.501616163848062,
|
|
"rewards/margins": 0.8756453422786004,
|
|
"rewards/rejected": -0.37402917843053834,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 10.05427074432373,
|
|
"kl": 18.795507431030273,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33660038.8982036,
|
|
"logits/rejected": -35006765.176470585,
|
|
"logps/chosen": -468.94217814371257,
|
|
"logps/rejected": -355.3689746732026,
|
|
"loss": 0.4573,
|
|
"loss/base_kto": 3.1093814373016357,
|
|
"loss/total_with_kl": 3.65879225730896,
|
|
"metrics/advantage_var": 329.9764709472656,
|
|
"regularization/advantage_var": 329.9764709472656,
|
|
"regularization/kl": 0.5494108200073242,
|
|
"rewards/chosen": 0.6614648510595995,
|
|
"rewards/margins": 0.8575095984059664,
|
|
"rewards/rejected": -0.19604474734636693,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 11.636170387268066,
|
|
"kl": 21.24412727355957,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34235893.71251932,
|
|
"logits/rejected": -34109161.75671406,
|
|
"logps/chosen": -474.5167117465224,
|
|
"logps/rejected": -366.46265304107425,
|
|
"loss": 0.4558,
|
|
"loss/base_kto": 3.141784906387329,
|
|
"loss/total_with_kl": 3.646454334259033,
|
|
"metrics/advantage_var": 303.104736328125,
|
|
"regularization/advantage_var": 303.104736328125,
|
|
"regularization/kl": 0.5046693682670593,
|
|
"rewards/chosen": 0.6264453162734255,
|
|
"rewards/margins": 0.8109377438643245,
|
|
"rewards/rejected": -0.18449242759089898,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 11.995245933532715,
|
|
"kl": 22.42620849609375,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34035581.96825397,
|
|
"logits/rejected": -36610854.596923076,
|
|
"logps/chosen": -469.57876984126983,
|
|
"logps/rejected": -351.3943269230769,
|
|
"loss": 0.4613,
|
|
"loss/base_kto": 3.067842721939087,
|
|
"loss/total_with_kl": 3.690115451812744,
|
|
"metrics/advantage_var": 373.7374572753906,
|
|
"regularization/advantage_var": 373.7374572753906,
|
|
"regularization/kl": 0.6222729086875916,
|
|
"rewards/chosen": 0.7026108999100943,
|
|
"rewards/margins": 0.9254983792520414,
|
|
"rewards/rejected": -0.2228874793419471,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 9.906083106994629,
|
|
"kl": 18.84529685974121,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35700720.43768997,
|
|
"logits/rejected": -35828755.75562701,
|
|
"logps/chosen": -509.5447378419453,
|
|
"logps/rejected": -365.9818127009646,
|
|
"loss": 0.4602,
|
|
"loss/base_kto": 3.1173386573791504,
|
|
"loss/total_with_kl": 3.681575059890747,
|
|
"metrics/advantage_var": 338.88067626953125,
|
|
"regularization/advantage_var": 338.88067626953125,
|
|
"regularization/kl": 0.5642362833023071,
|
|
"rewards/chosen": 0.6589171806729673,
|
|
"rewards/margins": 0.8846643167307321,
|
|
"rewards/rejected": -0.22574713605776478,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 13.125312805175781,
|
|
"kl": 20.307037353515625,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34938205.545864664,
|
|
"logits/rejected": -35919255.93495935,
|
|
"logps/chosen": -477.4031015037594,
|
|
"logps/rejected": -368.5592987804878,
|
|
"loss": 0.4881,
|
|
"loss/base_kto": 3.1182637214660645,
|
|
"loss/total_with_kl": 3.9045815467834473,
|
|
"metrics/advantage_var": 472.26300048828125,
|
|
"regularization/advantage_var": 472.26300048828125,
|
|
"regularization/kl": 0.7863179445266724,
|
|
"rewards/chosen": 0.6869242044319784,
|
|
"rewards/margins": 0.8638465637434621,
|
|
"rewards/rejected": -0.17692235931148373,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 9.469399452209473,
|
|
"kl": 16.510149002075195,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -33766469.341732286,
|
|
"logits/rejected": -35859425.835658915,
|
|
"logps/chosen": -485.0823818897638,
|
|
"logps/rejected": -369.6765019379845,
|
|
"loss": 0.4635,
|
|
"loss/base_kto": 3.0808887481689453,
|
|
"loss/total_with_kl": 3.707745313644409,
|
|
"metrics/advantage_var": 376.490478515625,
|
|
"regularization/advantage_var": 376.490478515625,
|
|
"regularization/kl": 0.6268566846847534,
|
|
"rewards/chosen": 0.6425019992618111,
|
|
"rewards/margins": 0.9252334739090373,
|
|
"rewards/rejected": -0.28273147464722626,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 15.960187911987305,
|
|
"kl": 20.4044246673584,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35527502.76923077,
|
|
"logits/rejected": -36641623.414634146,
|
|
"logps/chosen": -470.1169871794872,
|
|
"logps/rejected": -387.9233517530488,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.1373703479766846,
|
|
"loss/total_with_kl": 3.6938633918762207,
|
|
"metrics/advantage_var": 334.2300720214844,
|
|
"regularization/advantage_var": 334.2300720214844,
|
|
"regularization/kl": 0.5564931035041809,
|
|
"rewards/chosen": 0.6184520721435547,
|
|
"rewards/margins": 0.8400480689072027,
|
|
"rewards/rejected": -0.22159599676364805,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 21.24134635925293,
|
|
"kl": 19.327011108398438,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33151336.448,
|
|
"logits/rejected": -35577421.38625954,
|
|
"logps/chosen": -490.10015,
|
|
"logps/rejected": -381.9539599236641,
|
|
"loss": 0.4592,
|
|
"loss/base_kto": 3.028256893157959,
|
|
"loss/total_with_kl": 3.673821210861206,
|
|
"metrics/advantage_var": 387.72637939453125,
|
|
"regularization/advantage_var": 387.72637939453125,
|
|
"regularization/kl": 0.6455643773078918,
|
|
"rewards/chosen": 0.6580330078125,
|
|
"rewards/margins": 0.9728783419310592,
|
|
"rewards/rejected": -0.31484533411855914,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 12.932014465332031,
|
|
"kl": 16.66994285583496,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34448999.65644172,
|
|
"logits/rejected": -36296253.96178344,
|
|
"logps/chosen": -480.6158454754601,
|
|
"logps/rejected": -390.64965664808915,
|
|
"loss": 0.4593,
|
|
"loss/base_kto": 3.0751588344573975,
|
|
"loss/total_with_kl": 3.674107313156128,
|
|
"metrics/advantage_var": 359.7289733886719,
|
|
"regularization/advantage_var": 359.7289733886719,
|
|
"regularization/kl": 0.5989487171173096,
|
|
"rewards/chosen": 0.6390529702777511,
|
|
"rewards/margins": 0.9042419771074685,
|
|
"rewards/rejected": -0.26518900682971736,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 7.143278121948242,
|
|
"kl": 19.18073844909668,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35604486.131736524,
|
|
"logits/rejected": -35972671.58169935,
|
|
"logps/chosen": -461.79640718562877,
|
|
"logps/rejected": -395.61805555555554,
|
|
"loss": 0.4571,
|
|
"loss/base_kto": 3.1069934368133545,
|
|
"loss/total_with_kl": 3.656771183013916,
|
|
"metrics/advantage_var": 330.1967468261719,
|
|
"regularization/advantage_var": 330.1967468261719,
|
|
"regularization/kl": 0.5497775673866272,
|
|
"rewards/chosen": 0.6508044397045752,
|
|
"rewards/margins": 0.8794850690608099,
|
|
"rewards/rejected": -0.22868062935623468,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 26.480388641357422,
|
|
"kl": 20.91214370727539,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35775464.18604651,
|
|
"logits/rejected": -35679533.55590551,
|
|
"logps/chosen": -492.06598837209305,
|
|
"logps/rejected": -361.64721948818897,
|
|
"loss": 0.4646,
|
|
"loss/base_kto": 3.077267646789551,
|
|
"loss/total_with_kl": 3.7171669006347656,
|
|
"metrics/advantage_var": 384.3238220214844,
|
|
"regularization/advantage_var": 384.3238220214844,
|
|
"regularization/kl": 0.6398991942405701,
|
|
"rewards/chosen": 0.7391787506813227,
|
|
"rewards/margins": 0.9163473970657345,
|
|
"rewards/rejected": -0.1771686463844119,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 11.588574409484863,
|
|
"kl": 17.6380558013916,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34916281.166917294,
|
|
"logits/rejected": -35104245.177235775,
|
|
"logps/chosen": -487.8825187969925,
|
|
"logps/rejected": -395.2237042682927,
|
|
"loss": 0.4605,
|
|
"loss/base_kto": 3.077972173690796,
|
|
"loss/total_with_kl": 3.6838810443878174,
|
|
"metrics/advantage_var": 363.909423828125,
|
|
"regularization/advantage_var": 363.909423828125,
|
|
"regularization/kl": 0.6059091687202454,
|
|
"rewards/chosen": 0.6262954597186325,
|
|
"rewards/margins": 0.9238782442317803,
|
|
"rewards/rejected": -0.29758278451314785,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 14.420379638671875,
|
|
"kl": 20.456974029541016,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -32797281.295950156,
|
|
"logits/rejected": -36612009.329153605,
|
|
"logps/chosen": -502.29955218068534,
|
|
"logps/rejected": -359.5392829153605,
|
|
"loss": 0.462,
|
|
"loss/base_kto": 3.069866418838501,
|
|
"loss/total_with_kl": 3.6956498622894287,
|
|
"metrics/advantage_var": 375.845703125,
|
|
"regularization/advantage_var": 375.845703125,
|
|
"regularization/kl": 0.6257831454277039,
|
|
"rewards/chosen": 0.7031409718165887,
|
|
"rewards/margins": 0.9267154756524283,
|
|
"rewards/rejected": -0.22357450383583954,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 14.924396514892578,
|
|
"kl": 18.695880889892578,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34356465.12538226,
|
|
"logits/rejected": -35959634.60702875,
|
|
"logps/chosen": -459.5124235474006,
|
|
"logps/rejected": -376.58441493610223,
|
|
"loss": 0.4654,
|
|
"loss/base_kto": 3.134883165359497,
|
|
"loss/total_with_kl": 3.7233529090881348,
|
|
"metrics/advantage_var": 353.4350280761719,
|
|
"regularization/advantage_var": 353.4350280761719,
|
|
"regularization/kl": 0.5884692668914795,
|
|
"rewards/chosen": 0.6450473890392059,
|
|
"rewards/margins": 0.8598745396217113,
|
|
"rewards/rejected": -0.21482715058250548,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.060352325439453,
|
|
"kl": 20.296659469604492,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34791790.518053375,
|
|
"logits/rejected": -35867662.33281493,
|
|
"logps/chosen": -483.68754905808476,
|
|
"logps/rejected": -395.350651244168,
|
|
"loss": 0.4641,
|
|
"loss/base_kto": 3.0956616401672363,
|
|
"loss/total_with_kl": 3.712872266769409,
|
|
"metrics/advantage_var": 370.6968688964844,
|
|
"regularization/advantage_var": 370.6968688964844,
|
|
"regularization/kl": 0.6172103881835938,
|
|
"rewards/chosen": 0.6617858024369604,
|
|
"rewards/margins": 0.8896421907959708,
|
|
"rewards/rejected": -0.2278563883590105,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 10.029075622558594,
|
|
"kl": 21.600072860717773,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35024255.19496855,
|
|
"logits/rejected": -36128510.40993789,
|
|
"logps/chosen": -474.2742727987421,
|
|
"logps/rejected": -403.06766789596276,
|
|
"loss": 0.4711,
|
|
"loss/base_kto": 3.1070034503936768,
|
|
"loss/total_with_kl": 3.7690491676330566,
|
|
"metrics/advantage_var": 397.62518310546875,
|
|
"regularization/advantage_var": 397.62518310546875,
|
|
"regularization/kl": 0.662045955657959,
|
|
"rewards/chosen": 0.6627385361389544,
|
|
"rewards/margins": 0.8590032888290504,
|
|
"rewards/rejected": -0.19626475269009608,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 11.753561973571777,
|
|
"kl": 19.34547233581543,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -32902551.328050714,
|
|
"logits/rejected": -35807042.66255778,
|
|
"logps/chosen": -508.9552297939778,
|
|
"logps/rejected": -372.83573285824343,
|
|
"loss": 0.4666,
|
|
"loss/base_kto": 3.078979969024658,
|
|
"loss/total_with_kl": 3.7330939769744873,
|
|
"metrics/advantage_var": 392.8611755371094,
|
|
"regularization/advantage_var": 392.8611755371094,
|
|
"regularization/kl": 0.6541138887405396,
|
|
"rewards/chosen": 0.6873350793321612,
|
|
"rewards/margins": 0.9273398906705183,
|
|
"rewards/rejected": -0.24000481133835708,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 10.120428085327148,
|
|
"kl": 17.297834396362305,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34779033.90029325,
|
|
"logits/rejected": -35680735.46488294,
|
|
"logps/chosen": -478.8338526392962,
|
|
"logps/rejected": -380.67605560200667,
|
|
"loss": 0.468,
|
|
"loss/base_kto": 3.1093106269836426,
|
|
"loss/total_with_kl": 3.743926763534546,
|
|
"metrics/advantage_var": 381.1507873535156,
|
|
"regularization/advantage_var": 381.1507873535156,
|
|
"regularization/kl": 0.6346160769462585,
|
|
"rewards/chosen": 0.6901628153065432,
|
|
"rewards/margins": 0.8785802563780185,
|
|
"rewards/rejected": -0.18841744107147523,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.36275863647461,
|
|
"kl": 20.08066749572754,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34936683.87421384,
|
|
"logits/rejected": -36909545.73913044,
|
|
"logps/chosen": -496.70341981132077,
|
|
"logps/rejected": -388.51669254658384,
|
|
"loss": 0.4685,
|
|
"loss/base_kto": 3.0967319011688232,
|
|
"loss/total_with_kl": 3.747715711593628,
|
|
"metrics/advantage_var": 390.98138427734375,
|
|
"regularization/advantage_var": 390.98138427734375,
|
|
"regularization/kl": 0.650983989238739,
|
|
"rewards/chosen": 0.6852142046082694,
|
|
"rewards/margins": 0.9062081943513304,
|
|
"rewards/rejected": -0.22099398974306095,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 13.986465454101562,
|
|
"kl": 20.332181930541992,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34612885.81350482,
|
|
"logits/rejected": -37749831.58662614,
|
|
"logps/chosen": -509.0453175241158,
|
|
"logps/rejected": -334.9946333586626,
|
|
"loss": 0.4605,
|
|
"loss/base_kto": 3.1062123775482178,
|
|
"loss/total_with_kl": 3.6836297512054443,
|
|
"metrics/advantage_var": 346.79693603515625,
|
|
"regularization/advantage_var": 346.79693603515625,
|
|
"regularization/kl": 0.5774168372154236,
|
|
"rewards/chosen": 0.6701935893852994,
|
|
"rewards/margins": 0.8920773457967736,
|
|
"rewards/rejected": -0.22188375641147418,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 12.418783187866211,
|
|
"kl": 17.193639755249023,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34308268.8,
|
|
"logits/rejected": -35254160.0,
|
|
"logps/chosen": -462.082470703125,
|
|
"logps/rejected": -383.4199951171875,
|
|
"loss": 0.4642,
|
|
"loss/base_kto": 3.1278998851776123,
|
|
"loss/total_with_kl": 3.7134804725646973,
|
|
"metrics/advantage_var": 351.69970703125,
|
|
"regularization/advantage_var": 351.69970703125,
|
|
"regularization/kl": 0.585580050945282,
|
|
"rewards/chosen": 0.6585504055023194,
|
|
"rewards/margins": 0.8750317335128784,
|
|
"rewards/rejected": -0.21648132801055908,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 13.723422050476074,
|
|
"kl": 18.919214248657227,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34210896.75709779,
|
|
"logits/rejected": -36117884.433436535,
|
|
"logps/chosen": -487.7697160883281,
|
|
"logps/rejected": -371.83978328173373,
|
|
"loss": 0.4621,
|
|
"loss/base_kto": 3.0969226360321045,
|
|
"loss/total_with_kl": 3.696964979171753,
|
|
"metrics/advantage_var": 360.3858947753906,
|
|
"regularization/advantage_var": 360.3858947753906,
|
|
"regularization/kl": 0.6000425219535828,
|
|
"rewards/chosen": 0.6286294738576992,
|
|
"rewards/margins": 0.8873858929420887,
|
|
"rewards/rejected": -0.2587564190843895,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 10.867472648620605,
|
|
"kl": 18.517934799194336,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34305692.25974026,
|
|
"logits/rejected": -35613933.4939759,
|
|
"logps/chosen": -469.52450284090907,
|
|
"logps/rejected": -364.32163027108436,
|
|
"loss": 0.457,
|
|
"loss/base_kto": 3.108471632003784,
|
|
"loss/total_with_kl": 3.6561081409454346,
|
|
"metrics/advantage_var": 328.9109191894531,
|
|
"regularization/advantage_var": 328.9109191894531,
|
|
"regularization/kl": 0.5476366281509399,
|
|
"rewards/chosen": 0.6623576771129261,
|
|
"rewards/margins": 0.8936989816621779,
|
|
"rewards/rejected": -0.2313413045492517,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 10.805334091186523,
|
|
"kl": 17.159452438354492,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34961395.512195125,
|
|
"logits/rejected": -36792411.897435896,
|
|
"logps/chosen": -476.2725323932927,
|
|
"logps/rejected": -384.1209685496795,
|
|
"loss": 0.4627,
|
|
"loss/base_kto": 3.120931625366211,
|
|
"loss/total_with_kl": 3.7019202709198,
|
|
"metrics/advantage_var": 348.9420471191406,
|
|
"regularization/advantage_var": 348.9420471191406,
|
|
"regularization/kl": 0.5809884667396545,
|
|
"rewards/chosen": 0.642271274473609,
|
|
"rewards/margins": 0.9047954146604078,
|
|
"rewards/rejected": -0.26252414018679887,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 15.718250274658203,
|
|
"kl": 18.677274703979492,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34408862.8681672,
|
|
"logits/rejected": -35151037.86018237,
|
|
"logps/chosen": -500.32817524115757,
|
|
"logps/rejected": -386.14176481762917,
|
|
"loss": 0.4685,
|
|
"loss/base_kto": 3.0871524810791016,
|
|
"loss/total_with_kl": 3.7476539611816406,
|
|
"metrics/advantage_var": 396.69757080078125,
|
|
"regularization/advantage_var": 396.69757080078125,
|
|
"regularization/kl": 0.6605014204978943,
|
|
"rewards/chosen": 0.6487110788032556,
|
|
"rewards/margins": 0.9272517625379987,
|
|
"rewards/rejected": -0.27854068373474306,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 9.103044509887695,
|
|
"kl": 17.242324829101562,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34204503.17411402,
|
|
"logits/rejected": -36221067.56259905,
|
|
"logps/chosen": -444.0169010015408,
|
|
"logps/rejected": -366.1140303090333,
|
|
"loss": 0.4619,
|
|
"loss/base_kto": 3.1496777534484863,
|
|
"loss/total_with_kl": 3.6954143047332764,
|
|
"metrics/advantage_var": 327.7698974609375,
|
|
"regularization/advantage_var": 327.7698974609375,
|
|
"regularization/kl": 0.5457369089126587,
|
|
"rewards/chosen": 0.6019741346362192,
|
|
"rewards/margins": 0.8338497885152307,
|
|
"rewards/rejected": -0.2318756538790115,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 13.3783540725708,
|
|
"kl": 16.618896484375,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35433203.18057663,
|
|
"logits/rejected": -36360043.5942029,
|
|
"logps/chosen": -504.1375189681335,
|
|
"logps/rejected": -373.38315217391306,
|
|
"loss": 0.4627,
|
|
"loss/base_kto": 3.100325345993042,
|
|
"loss/total_with_kl": 3.7016799449920654,
|
|
"metrics/advantage_var": 361.1739196777344,
|
|
"regularization/advantage_var": 361.1739196777344,
|
|
"regularization/kl": 0.6013545393943787,
|
|
"rewards/chosen": 0.6479189327164738,
|
|
"rewards/margins": 0.9116225047179081,
|
|
"rewards/rejected": -0.2637035720014342,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 12.644377708435059,
|
|
"kl": 18.45366859436035,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33634692.6953748,
|
|
"logits/rejected": -35180525.18223584,
|
|
"logps/chosen": -475.40649920255186,
|
|
"logps/rejected": -367.3895482388974,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.1135945320129395,
|
|
"loss/total_with_kl": 3.6932201385498047,
|
|
"metrics/advantage_var": 348.12353515625,
|
|
"regularization/advantage_var": 348.12353515625,
|
|
"regularization/kl": 0.5796257257461548,
|
|
"rewards/chosen": 0.6570217978250847,
|
|
"rewards/margins": 0.8777010415350053,
|
|
"rewards/rejected": -0.22067924370992056,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 13.391596794128418,
|
|
"kl": 17.913007736206055,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36026576.7133758,
|
|
"logits/rejected": -36553376.19631902,
|
|
"logps/chosen": -481.25597133757964,
|
|
"logps/rejected": -394.75723734662574,
|
|
"loss": 0.467,
|
|
"loss/base_kto": 3.1105620861053467,
|
|
"loss/total_with_kl": 3.7358384132385254,
|
|
"metrics/advantage_var": 375.5413513183594,
|
|
"regularization/advantage_var": 375.5413513183594,
|
|
"regularization/kl": 0.6252763867378235,
|
|
"rewards/chosen": 0.6789868105748657,
|
|
"rewards/margins": 0.9045335713679782,
|
|
"rewards/rejected": -0.22554676079311253,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 13.19859504699707,
|
|
"kl": 17.68929672241211,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33718461.922720246,
|
|
"logits/rejected": -36536048.22748815,
|
|
"logps/chosen": -479.38591576506957,
|
|
"logps/rejected": -387.46608412322274,
|
|
"loss": 0.4602,
|
|
"loss/base_kto": 3.110572576522827,
|
|
"loss/total_with_kl": 3.6816444396972656,
|
|
"metrics/advantage_var": 342.98614501953125,
|
|
"regularization/advantage_var": 342.98614501953125,
|
|
"regularization/kl": 0.5710719227790833,
|
|
"rewards/chosen": 0.6081728338274005,
|
|
"rewards/margins": 0.8642510386648561,
|
|
"rewards/rejected": -0.2560782048374556,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 15.231182098388672,
|
|
"kl": 16.830408096313477,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34022834.32817338,
|
|
"logits/rejected": -36190615.01577287,
|
|
"logps/chosen": -478.59287925696594,
|
|
"logps/rejected": -363.70176951892745,
|
|
"loss": 0.4581,
|
|
"loss/base_kto": 3.0981926918029785,
|
|
"loss/total_with_kl": 3.664703130722046,
|
|
"metrics/advantage_var": 340.24658203125,
|
|
"regularization/advantage_var": 340.24658203125,
|
|
"regularization/kl": 0.5665105581283569,
|
|
"rewards/chosen": 0.6395979842903444,
|
|
"rewards/margins": 0.9017433546293379,
|
|
"rewards/rejected": -0.2621453703389935,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 8.46706771850586,
|
|
"kl": 18.756183624267578,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33320350.2394822,
|
|
"logits/rejected": -35217565.77643505,
|
|
"logps/chosen": -499.474767394822,
|
|
"logps/rejected": -375.9756655966767,
|
|
"loss": 0.4769,
|
|
"loss/base_kto": 3.155273199081421,
|
|
"loss/total_with_kl": 3.815504550933838,
|
|
"metrics/advantage_var": 396.5354919433594,
|
|
"regularization/advantage_var": 396.5354919433594,
|
|
"regularization/kl": 0.6602315902709961,
|
|
"rewards/chosen": 0.6185921480740544,
|
|
"rewards/margins": 0.8359393536596378,
|
|
"rewards/rejected": -0.21734720558558346,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_loss": 0.501307312482791,
|
|
"train_runtime": 11007.4603,
|
|
"train_samples_per_second": 13.465,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|