2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 28.284679412841797,
|
|
"kl": 6.050708770751953,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35905811.44303797,
|
|
"logits/rejected": -37529982.41975309,
|
|
"logps/chosen": -502.37598892405066,
|
|
"logps/rejected": -375.49351369598764,
|
|
"loss": 0.6245,
|
|
"loss/base_kto": 3.9251296520233154,
|
|
"metrics/advantage_var": 268.6701354980469,
|
|
"regularization/lipschitz_norm": 1106.2261962890625,
|
|
"regularization/w1": 1.0708268880844116,
|
|
"rewards/chosen": -0.017651963837539093,
|
|
"rewards/margins": 0.05768750654307916,
|
|
"rewards/rejected": -0.07533947038061825,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 19.63833236694336,
|
|
"kl": 15.056320190429688,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35845774.79198767,
|
|
"logits/rejected": -37855309.89540412,
|
|
"logps/chosen": -491.9004718798151,
|
|
"logps/rejected": -360.1343601426307,
|
|
"loss": 0.5916,
|
|
"loss/base_kto": 3.885058641433716,
|
|
"metrics/advantage_var": 169.64816284179688,
|
|
"regularization/lipschitz_norm": 875.93212890625,
|
|
"regularization/w1": 0.8479022979736328,
|
|
"rewards/chosen": 0.2674639544611903,
|
|
"rewards/margins": 0.08395554244312622,
|
|
"rewards/rejected": 0.1835084120180641,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 27.621639251708984,
|
|
"kl": 6.546723365783691,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34515715.28205128,
|
|
"logits/rejected": -35476376.97560976,
|
|
"logps/chosen": -469.22045272435895,
|
|
"logps/rejected": -363.3183117378049,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.901015043258667,
|
|
"metrics/advantage_var": 152.6851043701172,
|
|
"regularization/lipschitz_norm": 830.13037109375,
|
|
"regularization/w1": 0.8035661578178406,
|
|
"rewards/chosen": 0.1407624880472819,
|
|
"rewards/margins": 0.09935606010561066,
|
|
"rewards/rejected": 0.04140642794167123,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 21.418718338012695,
|
|
"kl": 8.766357421875,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37755302.5007874,
|
|
"logits/rejected": -37971259.931782946,
|
|
"logps/chosen": -505.62780511811025,
|
|
"logps/rejected": -384.92449127906974,
|
|
"loss": 0.5918,
|
|
"loss/base_kto": 3.8352959156036377,
|
|
"metrics/advantage_var": 188.04177856445312,
|
|
"regularization/lipschitz_norm": 929.0358276367188,
|
|
"regularization/w1": 0.8993067145347595,
|
|
"rewards/chosen": 0.1343505258635273,
|
|
"rewards/margins": 0.16536262182498715,
|
|
"rewards/rejected": -0.031012095961459846,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 18.957792282104492,
|
|
"kl": 7.623079776763916,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36767270.419601835,
|
|
"logits/rejected": -38095099.508771926,
|
|
"logps/chosen": -491.09331929555896,
|
|
"logps/rejected": -373.7985446570973,
|
|
"loss": 0.6075,
|
|
"loss/base_kto": 3.8539459705352783,
|
|
"metrics/advantage_var": 254.1329803466797,
|
|
"regularization/lipschitz_norm": 1039.2696533203125,
|
|
"regularization/w1": 1.006013035774231,
|
|
"rewards/chosen": 0.016018993091437207,
|
|
"rewards/margins": 0.16736023821641227,
|
|
"rewards/rejected": -0.15134124512497507,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 22.41180992126465,
|
|
"kl": 18.145795822143555,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -37671477.93587786,
|
|
"logits/rejected": -37557675.6224,
|
|
"logps/chosen": -491.1575381679389,
|
|
"logps/rejected": -360.1194,
|
|
"loss": 0.6012,
|
|
"loss/base_kto": 3.8942604064941406,
|
|
"metrics/advantage_var": 198.2205352783203,
|
|
"regularization/lipschitz_norm": 945.71484375,
|
|
"regularization/w1": 0.9154520034790039,
|
|
"rewards/chosen": 0.28311299331315604,
|
|
"rewards/margins": 0.09011754653581228,
|
|
"rewards/rejected": 0.19299544677734376,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 25.42111587524414,
|
|
"kl": 16.510923385620117,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -35904493.15337423,
|
|
"logits/rejected": -38749242.700636946,
|
|
"logps/chosen": -485.18131710122697,
|
|
"logps/rejected": -383.5733976910828,
|
|
"loss": 0.595,
|
|
"loss/base_kto": 3.810206174850464,
|
|
"metrics/advantage_var": 206.51370239257812,
|
|
"regularization/lipschitz_norm": 980.82373046875,
|
|
"regularization/w1": 0.9494373202323914,
|
|
"rewards/chosen": 0.229209314826076,
|
|
"rewards/margins": 0.1689713452635698,
|
|
"rewards/rejected": 0.06023796956250622,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 19.564281463623047,
|
|
"kl": 26.949377059936523,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -37505361.751124434,
|
|
"logits/rejected": -37022360.84828711,
|
|
"logps/chosen": -493.44958770614693,
|
|
"logps/rejected": -371.7893046492659,
|
|
"loss": 0.5864,
|
|
"loss/base_kto": 3.7792816162109375,
|
|
"metrics/advantage_var": 213.29966735839844,
|
|
"regularization/lipschitz_norm": 941.6534423828125,
|
|
"regularization/w1": 0.9115204215049744,
|
|
"rewards/chosen": 0.4891805806081334,
|
|
"rewards/margins": 0.17422503570856163,
|
|
"rewards/rejected": 0.3149555448995718,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 23.918167114257812,
|
|
"kl": 36.33083724975586,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35656098.14605067,
|
|
"logits/rejected": -36877939.17898194,
|
|
"logps/chosen": -463.679163561848,
|
|
"logps/rejected": -356.2293719211823,
|
|
"loss": 0.5872,
|
|
"loss/base_kto": 3.770932912826538,
|
|
"metrics/advantage_var": 219.7865753173828,
|
|
"regularization/lipschitz_norm": 957.6724853515625,
|
|
"regularization/w1": 0.9270269274711609,
|
|
"rewards/chosen": 0.6040142758592586,
|
|
"rewards/margins": 0.20233625003888195,
|
|
"rewards/rejected": 0.40167802582037665,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 17.953758239746094,
|
|
"kl": 32.3237190246582,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37548226.46884273,
|
|
"logits/rejected": -39389116.409240924,
|
|
"logps/chosen": -486.1354784866469,
|
|
"logps/rejected": -379.991852310231,
|
|
"loss": 0.5895,
|
|
"loss/base_kto": 3.783107042312622,
|
|
"metrics/advantage_var": 210.78970336914062,
|
|
"regularization/lipschitz_norm": 963.498046875,
|
|
"regularization/w1": 0.9326661229133606,
|
|
"rewards/chosen": 0.510858099962908,
|
|
"rewards/margins": 0.18344411924598863,
|
|
"rewards/rejected": 0.32741398071691935,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 22.509536743164062,
|
|
"kl": 27.011798858642578,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37872997.45025295,
|
|
"logits/rejected": -38662275.16739447,
|
|
"logps/chosen": -476.6013912310287,
|
|
"logps/rejected": -377.95146470160114,
|
|
"loss": 0.5996,
|
|
"loss/base_kto": 3.771259069442749,
|
|
"metrics/advantage_var": 262.2279052734375,
|
|
"regularization/lipschitz_norm": 1059.064697265625,
|
|
"regularization/w1": 1.0251744985580444,
|
|
"rewards/chosen": 0.42178815613307596,
|
|
"rewards/margins": 0.2127755734735281,
|
|
"rewards/rejected": 0.20901258265954786,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 18.072338104248047,
|
|
"kl": 10.750651359558105,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36808512.50713154,
|
|
"logits/rejected": -37048054.92758089,
|
|
"logps/chosen": -466.24187797147385,
|
|
"logps/rejected": -377.1495088597843,
|
|
"loss": 0.6019,
|
|
"loss/base_kto": 3.8303468227386475,
|
|
"metrics/advantage_var": 255.5202178955078,
|
|
"regularization/lipschitz_norm": 1017.7144775390625,
|
|
"regularization/w1": 0.9851474761962891,
|
|
"rewards/chosen": 0.050281687886136836,
|
|
"rewards/margins": 0.164875370079746,
|
|
"rewards/rejected": -0.11459368219360916,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 18.03236961364746,
|
|
"kl": 10.545516014099121,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37081604.78504673,
|
|
"logits/rejected": -37055478.369905956,
|
|
"logps/chosen": -478.5852316978193,
|
|
"logps/rejected": -377.98168103448273,
|
|
"loss": 0.5968,
|
|
"loss/base_kto": 3.806899309158325,
|
|
"metrics/advantage_var": 245.1384735107422,
|
|
"regularization/lipschitz_norm": 999.7939453125,
|
|
"regularization/w1": 0.9678005576133728,
|
|
"rewards/chosen": 0.1829061062536507,
|
|
"rewards/margins": 0.16642271267121994,
|
|
"rewards/rejected": 0.016483393582430752,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 17.374134063720703,
|
|
"kl": 17.86372947692871,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36710992.23263328,
|
|
"logits/rejected": -39415541.54311649,
|
|
"logps/chosen": -503.2741316639742,
|
|
"logps/rejected": -362.8714069591528,
|
|
"loss": 0.5849,
|
|
"loss/base_kto": 3.7193336486816406,
|
|
"metrics/advantage_var": 220.69456481933594,
|
|
"regularization/lipschitz_norm": 991.4678955078125,
|
|
"regularization/w1": 0.9597408175468445,
|
|
"rewards/chosen": 0.3227413343882522,
|
|
"rewards/margins": 0.23886214346141904,
|
|
"rewards/rejected": 0.08387919092683316,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 20.176193237304688,
|
|
"kl": 10.863235473632812,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36624831.61676647,
|
|
"logits/rejected": -38581535.79084967,
|
|
"logps/chosen": -461.8146519461078,
|
|
"logps/rejected": -373.11703431372547,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.756937026977539,
|
|
"metrics/advantage_var": 231.7496337890625,
|
|
"regularization/lipschitz_norm": 995.99560546875,
|
|
"regularization/w1": 0.9641237258911133,
|
|
"rewards/chosen": 0.20020303326452563,
|
|
"rewards/margins": 0.20762406086568042,
|
|
"rewards/rejected": -0.0074210276011548015,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 22.79453468322754,
|
|
"kl": 25.50826072692871,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37865885.570274636,
|
|
"logits/rejected": -38300373.01059002,
|
|
"logps/chosen": -520.7598445072698,
|
|
"logps/rejected": -392.857791225416,
|
|
"loss": 0.5893,
|
|
"loss/base_kto": 3.7324700355529785,
|
|
"metrics/advantage_var": 230.5997314453125,
|
|
"regularization/lipschitz_norm": 1014.0424194335938,
|
|
"regularization/w1": 0.9815930724143982,
|
|
"rewards/chosen": 0.4432368116733138,
|
|
"rewards/margins": 0.23891707849944135,
|
|
"rewards/rejected": 0.20431973317387245,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 19.35013771057129,
|
|
"kl": 19.88536262512207,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35062254.006389774,
|
|
"logits/rejected": -36604996.89296636,
|
|
"logps/chosen": -468.4251198083067,
|
|
"logps/rejected": -361.8530198776758,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7496471405029297,
|
|
"metrics/advantage_var": 272.22955322265625,
|
|
"regularization/lipschitz_norm": 1073.6064453125,
|
|
"regularization/w1": 1.0392509698867798,
|
|
"rewards/chosen": 0.3126804973370732,
|
|
"rewards/margins": 0.24065260263746324,
|
|
"rewards/rejected": 0.07202789469960998,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 24.394580841064453,
|
|
"kl": 10.459001541137695,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -35003234.84329564,
|
|
"logits/rejected": -36466824.32677761,
|
|
"logps/chosen": -494.7103190630049,
|
|
"logps/rejected": -363.49877080181545,
|
|
"loss": 0.585,
|
|
"loss/base_kto": 3.7073044776916504,
|
|
"metrics/advantage_var": 236.2416534423828,
|
|
"regularization/lipschitz_norm": 1005.060546875,
|
|
"regularization/w1": 0.9728986024856567,
|
|
"rewards/chosen": 0.19673321898033622,
|
|
"rewards/margins": 0.2668356047624136,
|
|
"rewards/rejected": -0.07010238578207735,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 19.257516860961914,
|
|
"kl": 8.215998649597168,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36699825.63265306,
|
|
"logits/rejected": -36798804.802488334,
|
|
"logps/chosen": -481.2975863422292,
|
|
"logps/rejected": -354.0599241835148,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.8107993602752686,
|
|
"metrics/advantage_var": 223.18716430664062,
|
|
"regularization/lipschitz_norm": 1012.6260986328125,
|
|
"regularization/w1": 0.9802221655845642,
|
|
"rewards/chosen": 0.09591743073994922,
|
|
"rewards/margins": 0.1644226304036958,
|
|
"rewards/rejected": -0.0685051996637466,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 19.96467399597168,
|
|
"kl": 9.938606262207031,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -37001703.61904762,
|
|
"logits/rejected": -36842392.024615385,
|
|
"logps/chosen": -480.93184523809526,
|
|
"logps/rejected": -387.9960817307692,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.7171807289123535,
|
|
"metrics/advantage_var": 260.2160949707031,
|
|
"regularization/lipschitz_norm": 1093.2796630859375,
|
|
"regularization/w1": 1.058294653892517,
|
|
"rewards/chosen": 0.14468497624472967,
|
|
"rewards/margins": 0.2707716485141107,
|
|
"rewards/rejected": -0.126086672269381,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 22.027286529541016,
|
|
"kl": 5.233973026275635,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37696667.05362776,
|
|
"logits/rejected": -38398868.21052632,
|
|
"logps/chosen": -478.69030954258676,
|
|
"logps/rejected": -359.3108552631579,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.75394606590271,
|
|
"metrics/advantage_var": 253.8077850341797,
|
|
"regularization/lipschitz_norm": 1010.2439575195312,
|
|
"regularization/w1": 0.9779161810874939,
|
|
"rewards/chosen": 0.05049951595462835,
|
|
"rewards/margins": 0.23183028833091027,
|
|
"rewards/rejected": -0.18133077237628192,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 21.019990921020508,
|
|
"kl": 5.614864349365234,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35586863.40740741,
|
|
"logits/rejected": -36523150.58227848,
|
|
"logps/chosen": -455.9927662037037,
|
|
"logps/rejected": -377.98484473892404,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.7866158485412598,
|
|
"metrics/advantage_var": 232.8219757080078,
|
|
"regularization/lipschitz_norm": 1007.7808837890625,
|
|
"regularization/w1": 0.9755318760871887,
|
|
"rewards/chosen": 0.03540201246002574,
|
|
"rewards/margins": 0.21647021557729978,
|
|
"rewards/rejected": -0.18106820311727403,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 24.934526443481445,
|
|
"kl": 8.18780517578125,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35661933.4351145,
|
|
"logits/rejected": -36096022.9376,
|
|
"logps/chosen": -466.3452290076336,
|
|
"logps/rejected": -357.3127,
|
|
"loss": 0.591,
|
|
"loss/base_kto": 3.6937973499298096,
|
|
"metrics/advantage_var": 254.6726837158203,
|
|
"regularization/lipschitz_norm": 1068.1162109375,
|
|
"regularization/w1": 1.0339365005493164,
|
|
"rewards/chosen": 0.16447889022244752,
|
|
"rewards/margins": 0.29996607283963506,
|
|
"rewards/rejected": -0.1354871826171875,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 23.624765396118164,
|
|
"kl": 17.251171112060547,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37303200.049155146,
|
|
"logits/rejected": -37838073.08108108,
|
|
"logps/chosen": -494.00633640552996,
|
|
"logps/rejected": -387.3216911764706,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.705303192138672,
|
|
"metrics/advantage_var": 290.14044189453125,
|
|
"regularization/lipschitz_norm": 1100.1011962890625,
|
|
"regularization/w1": 1.064897894859314,
|
|
"rewards/chosen": 0.3678935061218918,
|
|
"rewards/margins": 0.29070364064570503,
|
|
"rewards/rejected": 0.07718986547618678,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 19.30698585510254,
|
|
"kl": 18.461057662963867,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35688822.3655914,
|
|
"logits/rejected": -37567724.057233706,
|
|
"logps/chosen": -464.96370967741933,
|
|
"logps/rejected": -384.53885135135135,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.74275279045105,
|
|
"metrics/advantage_var": 268.9945983886719,
|
|
"regularization/lipschitz_norm": 1054.3267822265625,
|
|
"regularization/w1": 1.0205882787704468,
|
|
"rewards/chosen": 0.2288125971312164,
|
|
"rewards/margins": 0.2245003290090149,
|
|
"rewards/rejected": 0.004312268122201504,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 19.382633209228516,
|
|
"kl": 14.04481029510498,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36496845.93059937,
|
|
"logits/rejected": -37464602.94736842,
|
|
"logps/chosen": -464.81895701892745,
|
|
"logps/rejected": -378.781491873065,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.7514591217041016,
|
|
"metrics/advantage_var": 268.2990417480469,
|
|
"regularization/lipschitz_norm": 1063.7738037109375,
|
|
"regularization/w1": 1.0297330617904663,
|
|
"rewards/chosen": 0.17946819702533517,
|
|
"rewards/margins": 0.21765296234799383,
|
|
"rewards/rejected": -0.038184765322658666,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 29.22517204284668,
|
|
"kl": 19.706571578979492,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37008987.17808219,
|
|
"logits/rejected": -36886892.89245586,
|
|
"logps/chosen": -499.05493721461187,
|
|
"logps/rejected": -359.170044141252,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.6204020977020264,
|
|
"metrics/advantage_var": 321.3903503417969,
|
|
"regularization/lipschitz_norm": 1118.2493896484375,
|
|
"regularization/w1": 1.082465410232544,
|
|
"rewards/chosen": 0.4124527325913242,
|
|
"rewards/margins": 0.3715337486297566,
|
|
"rewards/rejected": 0.040918983961567644,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.34919548034668,
|
|
"kl": 17.98653221130371,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36831250.59001513,
|
|
"logits/rejected": -37122491.34733441,
|
|
"logps/chosen": -495.9337178517398,
|
|
"logps/rejected": -384.9459561793215,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.6959121227264404,
|
|
"metrics/advantage_var": 270.3876037597656,
|
|
"regularization/lipschitz_norm": 1078.271728515625,
|
|
"regularization/w1": 1.0437668561935425,
|
|
"rewards/chosen": 0.3510269649811484,
|
|
"rewards/margins": 0.2730278561535913,
|
|
"rewards/rejected": 0.07799910882755705,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 13.792253494262695,
|
|
"kl": 24.808151245117188,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37472919.7037037,
|
|
"logits/rejected": -38602560.81012658,
|
|
"logps/chosen": -503.2482638888889,
|
|
"logps/rejected": -366.41628757911394,
|
|
"loss": 0.5872,
|
|
"loss/base_kto": 3.6792945861816406,
|
|
"metrics/advantage_var": 238.5391082763672,
|
|
"regularization/lipschitz_norm": 1051.7332763671875,
|
|
"regularization/w1": 1.0180777311325073,
|
|
"rewards/chosen": 0.47948625352647567,
|
|
"rewards/margins": 0.25156523842684153,
|
|
"rewards/rejected": 0.2279210150996341,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 21.188846588134766,
|
|
"kl": 21.32284927368164,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37458432.0,
|
|
"logits/rejected": -38351492.8607595,
|
|
"logps/chosen": -522.0483217592592,
|
|
"logps/rejected": -363.99344837816454,
|
|
"loss": 0.6028,
|
|
"loss/base_kto": 3.698185682296753,
|
|
"metrics/advantage_var": 297.4457092285156,
|
|
"regularization/lipschitz_norm": 1161.6429443359375,
|
|
"regularization/w1": 1.1244704723358154,
|
|
"rewards/chosen": 0.41004020785108025,
|
|
"rewards/margins": 0.2871923972748466,
|
|
"rewards/rejected": 0.12284781057623369,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 23.26582145690918,
|
|
"kl": 11.471311569213867,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -34779942.2992126,
|
|
"logits/rejected": -36795117.34573644,
|
|
"logps/chosen": -476.6228346456693,
|
|
"logps/rejected": -383.8438468992248,
|
|
"loss": 0.5933,
|
|
"loss/base_kto": 3.767052412033081,
|
|
"metrics/advantage_var": 248.32920837402344,
|
|
"regularization/lipschitz_norm": 1012.0929565429688,
|
|
"regularization/w1": 0.9797059893608093,
|
|
"rewards/chosen": 0.10706037386195866,
|
|
"rewards/margins": 0.178669433277081,
|
|
"rewards/rejected": -0.07160905941512234,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 17.883386611938477,
|
|
"kl": 8.116411209106445,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35767321.221674874,
|
|
"logits/rejected": -38034070.318926975,
|
|
"logps/chosen": -481.5717877668309,
|
|
"logps/rejected": -369.3592585692995,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.750972032546997,
|
|
"metrics/advantage_var": 238.2175750732422,
|
|
"regularization/lipschitz_norm": 1017.2276611328125,
|
|
"regularization/w1": 0.9846763014793396,
|
|
"rewards/chosen": 0.014683341353593397,
|
|
"rewards/margins": 0.21594342625340476,
|
|
"rewards/rejected": -0.20126008489981137,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 18.580745697021484,
|
|
"kl": 8.209991455078125,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -35252204.85981309,
|
|
"logits/rejected": -35914681.37931035,
|
|
"logps/chosen": -455.03207749221184,
|
|
"logps/rejected": -386.3462970219436,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.736785888671875,
|
|
"metrics/advantage_var": 237.25537109375,
|
|
"regularization/lipschitz_norm": 1032.8397216796875,
|
|
"regularization/w1": 0.999788761138916,
|
|
"rewards/chosen": 0.07960102639837056,
|
|
"rewards/margins": 0.24826608421148283,
|
|
"rewards/rejected": -0.16866505781311225,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 26.838045120239258,
|
|
"kl": 20.73633575439453,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35850670.14420063,
|
|
"logits/rejected": -37429790.30529595,
|
|
"logps/chosen": -493.4886363636364,
|
|
"logps/rejected": -376.3583284657321,
|
|
"loss": 0.5871,
|
|
"loss/base_kto": 3.7232539653778076,
|
|
"metrics/advantage_var": 230.28018188476562,
|
|
"regularization/lipschitz_norm": 1005.8460083007812,
|
|
"regularization/w1": 0.9736589789390564,
|
|
"rewards/chosen": 0.352656636492212,
|
|
"rewards/margins": 0.232163707253444,
|
|
"rewards/rejected": 0.120492929238768,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 19.68084144592285,
|
|
"kl": 16.948158264160156,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35622512.07559055,
|
|
"logits/rejected": -36749321.5255814,
|
|
"logps/chosen": -486.8330216535433,
|
|
"logps/rejected": -344.28110465116276,
|
|
"loss": 0.5891,
|
|
"loss/base_kto": 3.764652729034424,
|
|
"metrics/advantage_var": 220.8280487060547,
|
|
"regularization/lipschitz_norm": 979.4452514648438,
|
|
"regularization/w1": 0.9481029510498047,
|
|
"rewards/chosen": 0.2265674500953494,
|
|
"rewards/margins": 0.18458749032650704,
|
|
"rewards/rejected": 0.041979959768842355,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 22.520645141601562,
|
|
"kl": 12.885197639465332,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36491045.58887172,
|
|
"logits/rejected": -37552917.434439175,
|
|
"logps/chosen": -500.4296271251932,
|
|
"logps/rejected": -379.19833629541864,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.756240129470825,
|
|
"metrics/advantage_var": 270.6735534667969,
|
|
"regularization/lipschitz_norm": 1071.513671875,
|
|
"regularization/w1": 1.0372251272201538,
|
|
"rewards/chosen": 0.1645966873286865,
|
|
"rewards/margins": 0.22212261738683284,
|
|
"rewards/rejected": -0.05752593005814635,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 18.744251251220703,
|
|
"kl": 11.187604904174805,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36985653.85113268,
|
|
"logits/rejected": -37628398.98489426,
|
|
"logps/chosen": -492.90564320388347,
|
|
"logps/rejected": -386.2225264350453,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.7459607124328613,
|
|
"metrics/advantage_var": 268.0847473144531,
|
|
"regularization/lipschitz_norm": 1077.4200439453125,
|
|
"regularization/w1": 1.0429426431655884,
|
|
"rewards/chosen": 0.13466049243717132,
|
|
"rewards/margins": 0.21679595512606684,
|
|
"rewards/rejected": -0.08213546268889552,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 17.9041748046875,
|
|
"kl": 6.419439792633057,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36052829.46031746,
|
|
"logits/rejected": -37627598.375384614,
|
|
"logps/chosen": -477.3235615079365,
|
|
"logps/rejected": -361.3248557692308,
|
|
"loss": 0.5911,
|
|
"loss/base_kto": 3.6789605617523193,
|
|
"metrics/advantage_var": 283.197998046875,
|
|
"regularization/lipschitz_norm": 1084.3193359375,
|
|
"regularization/w1": 1.0496209859848022,
|
|
"rewards/chosen": 0.046795109340122765,
|
|
"rewards/margins": 0.31432874910124053,
|
|
"rewards/rejected": -0.26753363976111777,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 18.287485122680664,
|
|
"kl": 5.721174716949463,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -34892882.84789644,
|
|
"logits/rejected": -35376807.56363636,
|
|
"logps/chosen": -468.4087783171521,
|
|
"logps/rejected": -388.48470643939396,
|
|
"loss": 0.605,
|
|
"loss/base_kto": 3.5825035572052,
|
|
"metrics/advantage_var": 404.2457580566406,
|
|
"regularization/lipschitz_norm": 1299.2603759765625,
|
|
"regularization/w1": 1.2576841115951538,
|
|
"rewards/chosen": 0.028945311759282083,
|
|
"rewards/margins": 0.44208738310753015,
|
|
"rewards/rejected": -0.4131420713482481,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 19.377307891845703,
|
|
"kl": 6.955938816070557,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36354910.48085758,
|
|
"logits/rejected": -37222311.80861244,
|
|
"logps/chosen": -465.1176301684533,
|
|
"logps/rejected": -387.9055023923445,
|
|
"loss": 0.6241,
|
|
"loss/base_kto": 3.3624539375305176,
|
|
"metrics/advantage_var": 646.35107421875,
|
|
"regularization/lipschitz_norm": 1684.5302734375,
|
|
"regularization/w1": 1.6306253671646118,
|
|
"rewards/chosen": 0.30485761329919603,
|
|
"rewards/margins": 0.7415017096128673,
|
|
"rewards/rejected": -0.43664409631367124,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 17.815826416015625,
|
|
"kl": 16.78569793701172,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35207618.81761006,
|
|
"logits/rejected": -36888296.149068326,
|
|
"logps/chosen": -443.0866745283019,
|
|
"logps/rejected": -391.5106997282609,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.3261260986328125,
|
|
"metrics/advantage_var": 559.1588134765625,
|
|
"regularization/lipschitz_norm": 1501.2880859375,
|
|
"regularization/w1": 1.4532469511032104,
|
|
"rewards/chosen": 0.4930549477631191,
|
|
"rewards/margins": 0.7203011078556802,
|
|
"rewards/rejected": -0.22724616009256113,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 18.48281478881836,
|
|
"kl": 5.493379592895508,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35686700.685154974,
|
|
"logits/rejected": -35851425.1994003,
|
|
"logps/chosen": -513.1646615008157,
|
|
"logps/rejected": -365.4383433283358,
|
|
"loss": 0.621,
|
|
"loss/base_kto": 3.372427463531494,
|
|
"metrics/advantage_var": 636.4617919921875,
|
|
"regularization/lipschitz_norm": 1647.95703125,
|
|
"regularization/w1": 1.5952223539352417,
|
|
"rewards/chosen": 0.19366044360311735,
|
|
"rewards/margins": 0.6924974813830556,
|
|
"rewards/rejected": -0.49883703777993815,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 21.839702606201172,
|
|
"kl": 7.518786907196045,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36309710.87675507,
|
|
"logits/rejected": -36580738.20344288,
|
|
"logps/chosen": -462.4204855694228,
|
|
"logps/rejected": -375.07868740219095,
|
|
"loss": 0.6256,
|
|
"loss/base_kto": 3.3311004638671875,
|
|
"metrics/advantage_var": 764.1552734375,
|
|
"regularization/lipschitz_norm": 1729.2777099609375,
|
|
"regularization/w1": 1.673940658569336,
|
|
"rewards/chosen": 0.2229258616145427,
|
|
"rewards/margins": 0.7866244048706068,
|
|
"rewards/rejected": -0.5636985432560642,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 16.18538475036621,
|
|
"kl": 11.524519920349121,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35388076.751145035,
|
|
"logits/rejected": -36500288.3072,
|
|
"logps/chosen": -462.506679389313,
|
|
"logps/rejected": -370.4149,
|
|
"loss": 0.606,
|
|
"loss/base_kto": 3.340698719024658,
|
|
"metrics/advantage_var": 555.2950439453125,
|
|
"regularization/lipschitz_norm": 1557.3011474609375,
|
|
"regularization/w1": 1.5074676275253296,
|
|
"rewards/chosen": 0.3492403685591603,
|
|
"rewards/margins": 0.7417162962935353,
|
|
"rewards/rejected": -0.392475927734375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.383241653442383,
|
|
"kl": 19.993188858032227,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -37304425.18429003,
|
|
"logits/rejected": -37189847.40453074,
|
|
"logps/chosen": -506.0303530966767,
|
|
"logps/rejected": -380.5348654935275,
|
|
"loss": 0.6178,
|
|
"loss/base_kto": 3.298877477645874,
|
|
"metrics/advantage_var": 695.72412109375,
|
|
"regularization/lipschitz_norm": 1697.6685791015625,
|
|
"regularization/w1": 1.6433429718017578,
|
|
"rewards/chosen": 0.687850168464407,
|
|
"rewards/margins": 0.7760547282350096,
|
|
"rewards/rejected": -0.08820455977060263,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 14.956389427185059,
|
|
"kl": 17.819286346435547,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34978176.0,
|
|
"logits/rejected": -35433167.23809524,
|
|
"logps/chosen": -487.08470394736844,
|
|
"logps/rejected": -374.43817429315476,
|
|
"loss": 0.5989,
|
|
"loss/base_kto": 3.218280792236328,
|
|
"metrics/advantage_var": 770.0177612304688,
|
|
"regularization/lipschitz_norm": 1625.3094482421875,
|
|
"regularization/w1": 1.5732994079589844,
|
|
"rewards/chosen": 0.6223972722103721,
|
|
"rewards/margins": 0.8812175334844374,
|
|
"rewards/rejected": -0.25882026127406527,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.838085174560547,
|
|
"kl": 7.458081245422363,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36180128.820512824,
|
|
"logits/rejected": -37325202.73170732,
|
|
"logps/chosen": -491.38296274038464,
|
|
"logps/rejected": -374.7822503810976,
|
|
"loss": 0.6124,
|
|
"loss/base_kto": 3.2906410694122314,
|
|
"metrics/advantage_var": 675.3164672851562,
|
|
"regularization/lipschitz_norm": 1661.6646728515625,
|
|
"regularization/w1": 1.60849130153656,
|
|
"rewards/chosen": 0.217655157431578,
|
|
"rewards/margins": 0.8089749161491847,
|
|
"rewards/rejected": -0.5913197587176067,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 22.3237247467041,
|
|
"kl": 6.157003879547119,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -36840332.105426356,
|
|
"logits/rejected": -38231676.97637795,
|
|
"logps/chosen": -494.1512112403101,
|
|
"logps/rejected": -381.05629921259845,
|
|
"loss": 0.6208,
|
|
"loss/base_kto": 3.332334280014038,
|
|
"metrics/advantage_var": 616.45263671875,
|
|
"regularization/lipschitz_norm": 1687.9783935546875,
|
|
"regularization/w1": 1.6339629888534546,
|
|
"rewards/chosen": 0.24477676273316376,
|
|
"rewards/margins": 0.7497021556650043,
|
|
"rewards/rejected": -0.5049253929318406,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 19.581022262573242,
|
|
"kl": 10.315558433532715,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35564502.52959502,
|
|
"logits/rejected": -36717654.670846395,
|
|
"logps/chosen": -476.301207165109,
|
|
"logps/rejected": -394.32102272727275,
|
|
"loss": 0.6127,
|
|
"loss/base_kto": 3.317326068878174,
|
|
"metrics/advantage_var": 617.7179565429688,
|
|
"regularization/lipschitz_norm": 1636.5614013671875,
|
|
"regularization/w1": 1.5841913223266602,
|
|
"rewards/chosen": 0.3656697971427181,
|
|
"rewards/margins": 0.7700343082798164,
|
|
"rewards/rejected": -0.40436451113709837,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 17.49793243408203,
|
|
"kl": 11.52022933959961,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34800088.36650869,
|
|
"logits/rejected": -34793354.88098918,
|
|
"logps/chosen": -471.1293937598736,
|
|
"logps/rejected": -405.65880989180835,
|
|
"loss": 0.6135,
|
|
"loss/base_kto": 3.3541908264160156,
|
|
"metrics/advantage_var": 601.1190795898438,
|
|
"regularization/lipschitz_norm": 1604.8447265625,
|
|
"regularization/w1": 1.5534896850585938,
|
|
"rewards/chosen": 0.3448968679419061,
|
|
"rewards/margins": 0.7352703691373272,
|
|
"rewards/rejected": -0.3903735011954212,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 17.19472885131836,
|
|
"kl": 7.987427711486816,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35549558.035928145,
|
|
"logits/rejected": -35133915.18954249,
|
|
"logps/chosen": -478.60235778443115,
|
|
"logps/rejected": -386.7259242238562,
|
|
"loss": 0.6233,
|
|
"loss/base_kto": 3.2980823516845703,
|
|
"metrics/advantage_var": 690.859375,
|
|
"regularization/lipschitz_norm": 1743.8031005859375,
|
|
"regularization/w1": 1.688001275062561,
|
|
"rewards/chosen": 0.3907644306114334,
|
|
"rewards/margins": 0.8247831329002917,
|
|
"rewards/rejected": -0.43401870228885825,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 29.313949584960938,
|
|
"kl": 23.783124923706055,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35311622.159398496,
|
|
"logits/rejected": -36611146.92682927,
|
|
"logps/chosen": -508.9872180451128,
|
|
"logps/rejected": -363.42146849593496,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.290377378463745,
|
|
"metrics/advantage_var": 632.463134765625,
|
|
"regularization/lipschitz_norm": 1579.3900146484375,
|
|
"regularization/w1": 1.528849482536316,
|
|
"rewards/chosen": 0.6658825408247181,
|
|
"rewards/margins": 0.7809848962855286,
|
|
"rewards/rejected": -0.11510235546081046,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.693548202514648,
|
|
"kl": 19.825727462768555,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35142020.07643312,
|
|
"logits/rejected": -35473009.0797546,
|
|
"logps/chosen": -492.3061305732484,
|
|
"logps/rejected": -376.4980828220859,
|
|
"loss": 0.615,
|
|
"loss/base_kto": 3.3185622692108154,
|
|
"metrics/advantage_var": 678.3754272460938,
|
|
"regularization/lipschitz_norm": 1654.5648193359375,
|
|
"regularization/w1": 1.601618766784668,
|
|
"rewards/chosen": 0.5552992122188495,
|
|
"rewards/margins": 0.7389632883862312,
|
|
"rewards/rejected": -0.18366407616738162,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 15.427619934082031,
|
|
"kl": 16.087003707885742,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -36531260.23529412,
|
|
"logits/rejected": -37808316.75576037,
|
|
"logps/chosen": -503.62808028616854,
|
|
"logps/rejected": -366.0517953149002,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.2703118324279785,
|
|
"metrics/advantage_var": 591.8820190429688,
|
|
"regularization/lipschitz_norm": 1569.50537109375,
|
|
"regularization/w1": 1.519281268119812,
|
|
"rewards/chosen": 0.5129609464272407,
|
|
"rewards/margins": 0.7680506315420881,
|
|
"rewards/rejected": -0.25508968511484736,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 16.78620719909668,
|
|
"kl": 12.485320091247559,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35135143.476635516,
|
|
"logits/rejected": -36348424.02507837,
|
|
"logps/chosen": -470.5177180685358,
|
|
"logps/rejected": -377.9007641065831,
|
|
"loss": 0.5899,
|
|
"loss/base_kto": 3.29404616355896,
|
|
"metrics/advantage_var": 570.2704467773438,
|
|
"regularization/lipschitz_norm": 1472.1197509765625,
|
|
"regularization/w1": 1.4250118732452393,
|
|
"rewards/chosen": 0.36283491853612976,
|
|
"rewards/margins": 0.7461272333106939,
|
|
"rewards/rejected": -0.38329231477456405,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 22.557743072509766,
|
|
"kl": 15.737161636352539,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35592294.56542811,
|
|
"logits/rejected": -36344102.341906205,
|
|
"logps/chosen": -470.34248788368336,
|
|
"logps/rejected": -377.56174357034797,
|
|
"loss": 0.5927,
|
|
"loss/base_kto": 3.2818260192871094,
|
|
"metrics/advantage_var": 525.9891967773438,
|
|
"regularization/lipschitz_norm": 1508.1322021484375,
|
|
"regularization/w1": 1.4598718881607056,
|
|
"rewards/chosen": 0.42693155069921246,
|
|
"rewards/margins": 0.7536140064805806,
|
|
"rewards/rejected": -0.3266824557813682,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.61176300048828,
|
|
"kl": 12.41404914855957,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34455428.27344992,
|
|
"logits/rejected": -35685320.94623656,
|
|
"logps/chosen": -485.0764109697933,
|
|
"logps/rejected": -376.02923387096774,
|
|
"loss": 0.5928,
|
|
"loss/base_kto": 3.223315954208374,
|
|
"metrics/advantage_var": 625.140625,
|
|
"regularization/lipschitz_norm": 1569.231689453125,
|
|
"regularization/w1": 1.5190162658691406,
|
|
"rewards/chosen": 0.4847335451547595,
|
|
"rewards/margins": 0.8737152368580285,
|
|
"rewards/rejected": -0.388981691703269,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 23.027080535888672,
|
|
"kl": 10.483132362365723,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34756435.95476575,
|
|
"logits/rejected": -35960261.906202726,
|
|
"logps/chosen": -483.3604604200323,
|
|
"logps/rejected": -373.5404926248109,
|
|
"loss": 0.6147,
|
|
"loss/base_kto": 3.2806389331817627,
|
|
"metrics/advantage_var": 656.40478515625,
|
|
"regularization/lipschitz_norm": 1691.227783203125,
|
|
"regularization/w1": 1.6371086835861206,
|
|
"rewards/chosen": 0.4133771116783623,
|
|
"rewards/margins": 0.8019734545182828,
|
|
"rewards/rejected": -0.38859634283992056,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 22.251970291137695,
|
|
"kl": 8.82465648651123,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -33834448.25723473,
|
|
"logits/rejected": -35144374.079027355,
|
|
"logps/chosen": -500.05114549839226,
|
|
"logps/rejected": -345.7993446048632,
|
|
"loss": 0.5984,
|
|
"loss/base_kto": 3.331987142562866,
|
|
"metrics/advantage_var": 516.9144897460938,
|
|
"regularization/lipschitz_norm": 1503.2855224609375,
|
|
"regularization/w1": 1.4551804065704346,
|
|
"rewards/chosen": 0.30820006342946143,
|
|
"rewards/margins": 0.7133809278505291,
|
|
"rewards/rejected": -0.40518086442106765,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 16.40721893310547,
|
|
"kl": 16.810983657836914,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34277114.294205055,
|
|
"logits/rejected": -37061586.45140033,
|
|
"logps/chosen": -503.2705237741456,
|
|
"logps/rejected": -363.0568883855025,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.2754509449005127,
|
|
"metrics/advantage_var": 777.3826293945312,
|
|
"regularization/lipschitz_norm": 1600.4676513671875,
|
|
"regularization/w1": 1.5492527484893799,
|
|
"rewards/chosen": 0.5745537631756129,
|
|
"rewards/margins": 0.7865318271203462,
|
|
"rewards/rejected": -0.2119780639447333,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 24.41370964050293,
|
|
"kl": 17.03086280822754,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34515247.0529595,
|
|
"logits/rejected": -36114839.67398119,
|
|
"logps/chosen": -462.88770443925233,
|
|
"logps/rejected": -375.2968994905956,
|
|
"loss": 0.6105,
|
|
"loss/base_kto": 3.323561906814575,
|
|
"metrics/advantage_var": 842.5131225585938,
|
|
"regularization/lipschitz_norm": 1611.8299560546875,
|
|
"regularization/w1": 1.5602514743804932,
|
|
"rewards/chosen": 0.5446985072435991,
|
|
"rewards/margins": 0.7637839136038068,
|
|
"rewards/rejected": -0.21908540636020768,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 15.436993598937988,
|
|
"kl": 16.175033569335938,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33907572.21587302,
|
|
"logits/rejected": -36099686.4,
|
|
"logps/chosen": -507.61944444444447,
|
|
"logps/rejected": -375.84088942307693,
|
|
"loss": 0.6094,
|
|
"loss/base_kto": 3.248197317123413,
|
|
"metrics/advantage_var": 731.9693603515625,
|
|
"regularization/lipschitz_norm": 1680.673095703125,
|
|
"regularization/w1": 1.6268917322158813,
|
|
"rewards/chosen": 0.5735631549169147,
|
|
"rewards/margins": 0.8389820908393906,
|
|
"rewards/rejected": -0.26541893592247595,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.70559310913086,
|
|
"kl": 15.646124839782715,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -33920421.64705882,
|
|
"logits/rejected": -35410454.80798771,
|
|
"logps/chosen": -482.28080286168523,
|
|
"logps/rejected": -355.8786242319508,
|
|
"loss": 0.6094,
|
|
"loss/base_kto": 3.2357337474823,
|
|
"metrics/advantage_var": 679.2140502929688,
|
|
"regularization/lipschitz_norm": 1693.5283203125,
|
|
"regularization/w1": 1.63933527469635,
|
|
"rewards/chosen": 0.5108369909143978,
|
|
"rewards/margins": 0.8512766306138386,
|
|
"rewards/rejected": -0.34043963969944074,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 17.28391456604004,
|
|
"kl": 12.305365562438965,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34946248.94191523,
|
|
"logits/rejected": -35778247.86314152,
|
|
"logps/chosen": -473.3879513343799,
|
|
"logps/rejected": -369.0113724727838,
|
|
"loss": 0.6143,
|
|
"loss/base_kto": 3.30818247795105,
|
|
"metrics/advantage_var": 636.6775512695312,
|
|
"regularization/lipschitz_norm": 1659.657470703125,
|
|
"regularization/w1": 1.606548547744751,
|
|
"rewards/chosen": 0.39057000128777475,
|
|
"rewards/margins": 0.7394870117003913,
|
|
"rewards/rejected": -0.34891701041261663,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 17.96364402770996,
|
|
"kl": 12.297408103942871,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33265059.41267388,
|
|
"logits/rejected": -34707653.358609796,
|
|
"logps/chosen": -470.75062789799074,
|
|
"logps/rejected": -368.9062746840442,
|
|
"loss": 0.6021,
|
|
"loss/base_kto": 3.266735792160034,
|
|
"metrics/advantage_var": 570.1412963867188,
|
|
"regularization/lipschitz_norm": 1601.023681640625,
|
|
"regularization/w1": 1.5497907400131226,
|
|
"rewards/chosen": 0.41282887289291925,
|
|
"rewards/margins": 0.7918910132386809,
|
|
"rewards/rejected": -0.37906214034576174,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 14.205744743347168,
|
|
"kl": 12.191987991333008,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34773849.766233765,
|
|
"logits/rejected": -35572643.469879515,
|
|
"logps/chosen": -484.36434659090907,
|
|
"logps/rejected": -386.9334760918675,
|
|
"loss": 0.6069,
|
|
"loss/base_kto": 3.2693467140197754,
|
|
"metrics/advantage_var": 635.6256103515625,
|
|
"regularization/lipschitz_norm": 1638.4261474609375,
|
|
"regularization/w1": 1.5859966278076172,
|
|
"rewards/chosen": 0.4410676336907721,
|
|
"rewards/margins": 0.8002055709378071,
|
|
"rewards/rejected": -0.359137937247035,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 24.168249130249023,
|
|
"kl": 6.707913398742676,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34532083.84686064,
|
|
"logits/rejected": -35766373.25677831,
|
|
"logps/chosen": -484.91089203675347,
|
|
"logps/rejected": -375.4888855661882,
|
|
"loss": 0.6015,
|
|
"loss/base_kto": 3.3565971851348877,
|
|
"metrics/advantage_var": 582.0673828125,
|
|
"regularization/lipschitz_norm": 1503.3599853515625,
|
|
"regularization/w1": 1.4552525281906128,
|
|
"rewards/chosen": 0.24142561207138927,
|
|
"rewards/margins": 0.7245304578150097,
|
|
"rewards/rejected": -0.4831048457436204,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 21.05829620361328,
|
|
"kl": 14.777182579040527,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34122390.773006134,
|
|
"logits/rejected": -35606338.85350318,
|
|
"logps/chosen": -483.98557323619633,
|
|
"logps/rejected": -382.7797074044586,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.2743585109710693,
|
|
"metrics/advantage_var": 496.87646484375,
|
|
"regularization/lipschitz_norm": 1477.9449462890625,
|
|
"regularization/w1": 1.4306507110595703,
|
|
"rewards/chosen": 0.47881073746944497,
|
|
"rewards/margins": 0.7498130999576267,
|
|
"rewards/rejected": -0.2710023624881817,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 23.042417526245117,
|
|
"kl": 16.886045455932617,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34682008.715855576,
|
|
"logits/rejected": -35531987.80715396,
|
|
"logps/chosen": -473.9014423076923,
|
|
"logps/rejected": -392.2607163685848,
|
|
"loss": 0.5944,
|
|
"loss/base_kto": 3.3150672912597656,
|
|
"metrics/advantage_var": 544.8585205078125,
|
|
"regularization/lipschitz_norm": 1487.87109375,
|
|
"regularization/w1": 1.4402592182159424,
|
|
"rewards/chosen": 0.4643519235368426,
|
|
"rewards/margins": 0.7374791219415374,
|
|
"rewards/rejected": -0.2731271984046948,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 19.28875160217285,
|
|
"kl": 11.619124412536621,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34951866.11161388,
|
|
"logits/rejected": -35454730.37277147,
|
|
"logps/chosen": -473.4477281297134,
|
|
"logps/rejected": -349.80080024311184,
|
|
"loss": 0.6135,
|
|
"loss/base_kto": 3.3116226196289062,
|
|
"metrics/advantage_var": 611.7669067382812,
|
|
"regularization/lipschitz_norm": 1648.8114013671875,
|
|
"regularization/w1": 1.5960493087768555,
|
|
"rewards/chosen": 0.49080168895232373,
|
|
"rewards/margins": 0.758468399012975,
|
|
"rewards/rejected": -0.26766671006065135,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 19.19460105895996,
|
|
"kl": 9.19257640838623,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34895287.32376395,
|
|
"logits/rejected": -35518238.186830014,
|
|
"logps/chosen": -485.94836523125997,
|
|
"logps/rejected": -359.33489663093417,
|
|
"loss": 0.6081,
|
|
"loss/base_kto": 3.403186082839966,
|
|
"metrics/advantage_var": 565.5280151367188,
|
|
"regularization/lipschitz_norm": 1509.8929443359375,
|
|
"regularization/w1": 1.4615763425827026,
|
|
"rewards/chosen": 0.2992124420604067,
|
|
"rewards/margins": 0.6448627571625793,
|
|
"rewards/rejected": -0.34565031510217264,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 16.687786102294922,
|
|
"kl": 13.650378227233887,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -33104306.13824885,
|
|
"logits/rejected": -34119373.93958665,
|
|
"logps/chosen": -476.690428187404,
|
|
"logps/rejected": -352.6248012718601,
|
|
"loss": 0.5888,
|
|
"loss/base_kto": 3.275512456893921,
|
|
"metrics/advantage_var": 519.4085693359375,
|
|
"regularization/lipschitz_norm": 1482.2254638671875,
|
|
"regularization/w1": 1.434794306755066,
|
|
"rewards/chosen": 0.44653048420098884,
|
|
"rewards/margins": 0.7729975888717211,
|
|
"rewards/rejected": -0.3264671046707323,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.426610946655273,
|
|
"kl": 12.734430313110352,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34698978.6749226,
|
|
"logits/rejected": -35704993.514195584,
|
|
"logps/chosen": -469.3546826625387,
|
|
"logps/rejected": -393.7551754731861,
|
|
"loss": 0.6159,
|
|
"loss/base_kto": 3.3413281440734863,
|
|
"metrics/advantage_var": 599.4804077148438,
|
|
"regularization/lipschitz_norm": 1638.5653076171875,
|
|
"regularization/w1": 1.58613121509552,
|
|
"rewards/chosen": 0.47008908865252513,
|
|
"rewards/margins": 0.7260038188809853,
|
|
"rewards/rejected": -0.2559147302284602,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 20.585351943969727,
|
|
"kl": 12.331831932067871,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34554958.76923077,
|
|
"logits/rejected": -35367734.450793654,
|
|
"logps/chosen": -495.54475961538463,
|
|
"logps/rejected": -391.78018353174605,
|
|
"loss": 0.6025,
|
|
"loss/base_kto": 3.263157606124878,
|
|
"metrics/advantage_var": 576.9074096679688,
|
|
"regularization/lipschitz_norm": 1608.2530517578125,
|
|
"regularization/w1": 1.5567890405654907,
|
|
"rewards/chosen": 0.4312304217998798,
|
|
"rewards/margins": 0.7955246131906288,
|
|
"rewards/rejected": -0.36429419139074903,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 22.021099090576172,
|
|
"kl": 13.9457426071167,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35271970.24203822,
|
|
"logits/rejected": -37426568.63803681,
|
|
"logps/chosen": -474.9701433121019,
|
|
"logps/rejected": -371.26303680981596,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.254892110824585,
|
|
"metrics/advantage_var": 550.8092041015625,
|
|
"regularization/lipschitz_norm": 1523.3638916015625,
|
|
"regularization/w1": 1.474616289138794,
|
|
"rewards/chosen": 0.5267801467020801,
|
|
"rewards/margins": 0.8088675780656129,
|
|
"rewards/rejected": -0.28208743136353287,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 20.054548263549805,
|
|
"kl": 13.417973518371582,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34051679.255813956,
|
|
"logits/rejected": -34958796.10810811,
|
|
"logps/chosen": -478.068359375,
|
|
"logps/rejected": -375.1455078125,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.3483033180236816,
|
|
"metrics/advantage_var": 570.8318481445312,
|
|
"regularization/lipschitz_norm": 1520.9615478515625,
|
|
"regularization/w1": 1.4722908735275269,
|
|
"rewards/chosen": 0.5058717505876408,
|
|
"rewards/margins": 0.7245585503329427,
|
|
"rewards/rejected": -0.21868679974530195,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 18.050657272338867,
|
|
"kl": 18.97049331665039,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -36081286.308426075,
|
|
"logits/rejected": -36586494.42703533,
|
|
"logps/chosen": -485.46244038155805,
|
|
"logps/rejected": -387.92660330261134,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.316699981689453,
|
|
"metrics/advantage_var": 575.1785888671875,
|
|
"regularization/lipschitz_norm": 1487.2132568359375,
|
|
"regularization/w1": 1.439622402191162,
|
|
"rewards/chosen": 0.591605327466713,
|
|
"rewards/margins": 0.7336290732364423,
|
|
"rewards/rejected": -0.14202374576972926,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 16.62051010131836,
|
|
"kl": 13.945266723632812,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -33998118.48366013,
|
|
"logits/rejected": -35030258.93093093,
|
|
"logps/chosen": -488.01485906862746,
|
|
"logps/rejected": -376.11674174174175,
|
|
"loss": 0.569,
|
|
"loss/base_kto": 3.2626521587371826,
|
|
"metrics/advantage_var": 449.40869140625,
|
|
"regularization/lipschitz_norm": 1331.864501953125,
|
|
"regularization/w1": 1.2892446517944336,
|
|
"rewards/chosen": 0.4739050771675858,
|
|
"rewards/margins": 0.7584942317518575,
|
|
"rewards/rejected": -0.28458915458427175,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 20.428098678588867,
|
|
"kl": 10.758501052856445,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34294731.0605613,
|
|
"logits/rejected": -34568945.99004975,
|
|
"logps/chosen": -459.1706979320532,
|
|
"logps/rejected": -383.0700404228856,
|
|
"loss": 0.5571,
|
|
"loss/base_kto": 3.2008590698242188,
|
|
"metrics/advantage_var": 456.99053955078125,
|
|
"regularization/lipschitz_norm": 1297.861572265625,
|
|
"regularization/w1": 1.256330132484436,
|
|
"rewards/chosen": 0.45935392168459194,
|
|
"rewards/margins": 0.8914253860586849,
|
|
"rewards/rejected": -0.43207146437409305,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 14.740876197814941,
|
|
"kl": 14.42285442352295,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33524542.79245283,
|
|
"logits/rejected": -35885968.69565217,
|
|
"logps/chosen": -498.733981918239,
|
|
"logps/rejected": -361.85978746118013,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.1709067821502686,
|
|
"metrics/advantage_var": 468.06396484375,
|
|
"regularization/lipschitz_norm": 1341.8985595703125,
|
|
"regularization/w1": 1.2989577054977417,
|
|
"rewards/chosen": 0.5989355891005798,
|
|
"rewards/margins": 0.8622489299451547,
|
|
"rewards/rejected": -0.2633133408445749,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 14.413639068603516,
|
|
"kl": 14.247815132141113,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34089283.03828484,
|
|
"logits/rejected": -36174500.13397129,
|
|
"logps/chosen": -483.3604517611026,
|
|
"logps/rejected": -385.9978568580542,
|
|
"loss": 0.5572,
|
|
"loss/base_kto": 3.187278985977173,
|
|
"metrics/advantage_var": 452.8713073730469,
|
|
"regularization/lipschitz_norm": 1312.0400390625,
|
|
"regularization/w1": 1.2700546979904175,
|
|
"rewards/chosen": 0.5540032145805417,
|
|
"rewards/margins": 0.8576783292137156,
|
|
"rewards/rejected": -0.30367511463317387,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 16.39202117919922,
|
|
"kl": 14.53508472442627,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34614318.618524335,
|
|
"logits/rejected": -34455934.20839813,
|
|
"logps/chosen": -482.4339678178964,
|
|
"logps/rejected": -374.2196734059098,
|
|
"loss": 0.5606,
|
|
"loss/base_kto": 3.2024028301239014,
|
|
"metrics/advantage_var": 450.5284118652344,
|
|
"regularization/lipschitz_norm": 1324.870361328125,
|
|
"regularization/w1": 1.2824745178222656,
|
|
"rewards/chosen": 0.5422733133217229,
|
|
"rewards/margins": 0.8314268439824442,
|
|
"rewards/rejected": -0.28915353066072125,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.80988121032715,
|
|
"kl": 12.371950149536133,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33979697.87012987,
|
|
"logits/rejected": -34665212.915662654,
|
|
"logps/chosen": -472.13012378246754,
|
|
"logps/rejected": -371.0192724021084,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.1427133083343506,
|
|
"metrics/advantage_var": 495.1771545410156,
|
|
"regularization/lipschitz_norm": 1394.9569091796875,
|
|
"regularization/w1": 1.350318193435669,
|
|
"rewards/chosen": 0.5426324621423498,
|
|
"rewards/margins": 0.913743010598381,
|
|
"rewards/rejected": -0.37111054845603114,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 13.9981107711792,
|
|
"kl": 8.827503204345703,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33974710.85714286,
|
|
"logits/rejected": -36571425.81132075,
|
|
"logps/chosen": -490.1635287267081,
|
|
"logps/rejected": -362.41951650943395,
|
|
"loss": 0.5635,
|
|
"loss/base_kto": 3.199734926223755,
|
|
"metrics/advantage_var": 461.0420837402344,
|
|
"regularization/lipschitz_norm": 1351.3529052734375,
|
|
"regularization/w1": 1.3081096410751343,
|
|
"rewards/chosen": 0.4198208447568905,
|
|
"rewards/margins": 0.8600601332789031,
|
|
"rewards/rejected": -0.4402392885220126,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 19.348628997802734,
|
|
"kl": 12.69625473022461,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34444501.8021978,
|
|
"logits/rejected": -36621077.897356145,
|
|
"logps/chosen": -524.8705357142857,
|
|
"logps/rejected": -371.7057737169518,
|
|
"loss": 0.559,
|
|
"loss/base_kto": 3.1831517219543457,
|
|
"metrics/advantage_var": 447.22412109375,
|
|
"regularization/lipschitz_norm": 1331.1708984375,
|
|
"regularization/w1": 1.2885735034942627,
|
|
"rewards/chosen": 0.534057042288069,
|
|
"rewards/margins": 0.8541402570315809,
|
|
"rewards/rejected": -0.32008321474351187,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 12.006818771362305,
|
|
"kl": 13.238250732421875,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33977827.0778626,
|
|
"logits/rejected": -35741846.7328,
|
|
"logps/chosen": -473.8763835877863,
|
|
"logps/rejected": -391.372475,
|
|
"loss": 0.561,
|
|
"loss/base_kto": 3.1720783710479736,
|
|
"metrics/advantage_var": 465.51226806640625,
|
|
"regularization/lipschitz_norm": 1359.76171875,
|
|
"regularization/w1": 1.3162492513656616,
|
|
"rewards/chosen": 0.5685190914241413,
|
|
"rewards/margins": 0.8942543453303913,
|
|
"rewards/rejected": -0.32573525390625,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 23.530920028686523,
|
|
"kl": 8.311408042907715,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33926570.79694656,
|
|
"logits/rejected": -35590327.5008,
|
|
"logps/chosen": -493.20209923664123,
|
|
"logps/rejected": -365.7894,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.1930572986602783,
|
|
"metrics/advantage_var": 473.5223693847656,
|
|
"regularization/lipschitz_norm": 1309.0533447265625,
|
|
"regularization/w1": 1.267163634300232,
|
|
"rewards/chosen": 0.47187770231989506,
|
|
"rewards/margins": 0.900834831226145,
|
|
"rewards/rejected": -0.42895712890625,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 13.836443901062012,
|
|
"kl": 11.999221801757812,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -33998593.96018377,
|
|
"logits/rejected": -34455081.64593302,
|
|
"logps/chosen": -444.62433001531394,
|
|
"logps/rejected": -373.3021331738437,
|
|
"loss": 0.5521,
|
|
"loss/base_kto": 3.234546661376953,
|
|
"metrics/advantage_var": 380.58978271484375,
|
|
"regularization/lipschitz_norm": 1221.0994873046875,
|
|
"regularization/w1": 1.182024359703064,
|
|
"rewards/chosen": 0.4462666299770291,
|
|
"rewards/margins": 0.7969859784525325,
|
|
"rewards/rejected": -0.3507193484755034,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 18.7479305267334,
|
|
"kl": 14.273015022277832,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -35007398.67912772,
|
|
"logits/rejected": -35545569.504702196,
|
|
"logps/chosen": -487.9390576323988,
|
|
"logps/rejected": -392.7202684169279,
|
|
"loss": 0.5461,
|
|
"loss/base_kto": 3.14363169670105,
|
|
"metrics/advantage_var": 409.0979309082031,
|
|
"regularization/lipschitz_norm": 1265.660888671875,
|
|
"regularization/w1": 1.2251596450805664,
|
|
"rewards/chosen": 0.5476035239911896,
|
|
"rewards/margins": 0.8756958539193097,
|
|
"rewards/rejected": -0.3280923299281201,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 17.080482482910156,
|
|
"kl": 16.26753044128418,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35982050.93623639,
|
|
"logits/rejected": -34576348.63422292,
|
|
"logps/chosen": -488.40629860031106,
|
|
"logps/rejected": -367.37259615384613,
|
|
"loss": 0.5539,
|
|
"loss/base_kto": 3.173492193222046,
|
|
"metrics/advantage_var": 420.8414001464844,
|
|
"regularization/lipschitz_norm": 1298.9735107421875,
|
|
"regularization/w1": 1.2574061155319214,
|
|
"rewards/chosen": 0.5900054504489454,
|
|
"rewards/margins": 0.8648827621195572,
|
|
"rewards/rejected": -0.27487731167061175,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 17.228593826293945,
|
|
"kl": 16.6585693359375,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34466087.35340729,
|
|
"logits/rejected": -36132488.4807396,
|
|
"logps/chosen": -464.84429477020603,
|
|
"logps/rejected": -355.222626155624,
|
|
"loss": 0.5592,
|
|
"loss/base_kto": 3.240309953689575,
|
|
"metrics/advantage_var": 381.2550048828125,
|
|
"regularization/lipschitz_norm": 1273.9112548828125,
|
|
"regularization/w1": 1.233146071434021,
|
|
"rewards/chosen": 0.4801065048967413,
|
|
"rewards/margins": 0.7706964590554831,
|
|
"rewards/rejected": -0.29058995415874184,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 17.9788818359375,
|
|
"kl": 13.090388298034668,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34962850.98262243,
|
|
"logits/rejected": -35811388.14219475,
|
|
"logps/chosen": -455.1439079778831,
|
|
"logps/rejected": -396.12533809891806,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.1894686222076416,
|
|
"metrics/advantage_var": 433.10406494140625,
|
|
"regularization/lipschitz_norm": 1286.2496337890625,
|
|
"regularization/w1": 1.2450895309448242,
|
|
"rewards/chosen": 0.5177260846323312,
|
|
"rewards/margins": 0.8659712248377263,
|
|
"rewards/rejected": -0.3482451402053951,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 18.649703979492188,
|
|
"kl": 11.21081256866455,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35445656.485225506,
|
|
"logits/rejected": -35233978.474097334,
|
|
"logps/chosen": -472.22851866251943,
|
|
"logps/rejected": -393.0775363029827,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.2486932277679443,
|
|
"metrics/advantage_var": 397.58642578125,
|
|
"regularization/lipschitz_norm": 1267.3721923828125,
|
|
"regularization/w1": 1.226816177368164,
|
|
"rewards/chosen": 0.4129372414311577,
|
|
"rewards/margins": 0.77915225113546,
|
|
"rewards/rejected": -0.3662150097043024,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 15.14542293548584,
|
|
"kl": 12.136643409729004,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -31783607.426791277,
|
|
"logits/rejected": -35130640.852664575,
|
|
"logps/chosen": -467.41111760124613,
|
|
"logps/rejected": -386.44278996865205,
|
|
"loss": 0.5635,
|
|
"loss/base_kto": 3.24726939201355,
|
|
"metrics/advantage_var": 387.1895751953125,
|
|
"regularization/lipschitz_norm": 1302.4925537109375,
|
|
"regularization/w1": 1.260812759399414,
|
|
"rewards/chosen": 0.4486684353552132,
|
|
"rewards/margins": 0.7824724792117473,
|
|
"rewards/rejected": -0.3338040438565341,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 19.20258903503418,
|
|
"kl": 12.994461059570312,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -33757476.571428575,
|
|
"logits/rejected": -35397613.29680365,
|
|
"logps/chosen": -474.8577447833066,
|
|
"logps/rejected": -384.66799847793,
|
|
"loss": 0.5644,
|
|
"loss/base_kto": 3.19708514213562,
|
|
"metrics/advantage_var": 454.6419982910156,
|
|
"regularization/lipschitz_norm": 1361.6043701171875,
|
|
"regularization/w1": 1.3180330991744995,
|
|
"rewards/chosen": 0.49236041661441615,
|
|
"rewards/margins": 0.8630950160602875,
|
|
"rewards/rejected": -0.3707345994458714,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 15.626067161560059,
|
|
"kl": 13.845202445983887,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35151315.31902718,
|
|
"logits/rejected": -35724917.20481928,
|
|
"logps/chosen": -481.55767167381975,
|
|
"logps/rejected": -388.76947074010326,
|
|
"loss": 0.5637,
|
|
"loss/base_kto": 3.205005407333374,
|
|
"metrics/advantage_var": 493.1046447753906,
|
|
"regularization/lipschitz_norm": 1347.6864013671875,
|
|
"regularization/w1": 1.3045603036880493,
|
|
"rewards/chosen": 0.5400235635869323,
|
|
"rewards/margins": 0.8575986473343102,
|
|
"rewards/rejected": -0.3175750837473779,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 18.5164737701416,
|
|
"kl": 17.44912338256836,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -35019616.415584415,
|
|
"logits/rejected": -36521962.409638554,
|
|
"logps/chosen": -530.0438311688312,
|
|
"logps/rejected": -361.7835560993976,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.148027181625366,
|
|
"metrics/advantage_var": 465.3861389160156,
|
|
"regularization/lipschitz_norm": 1375.33935546875,
|
|
"regularization/w1": 1.3313285112380981,
|
|
"rewards/chosen": 0.585678397835075,
|
|
"rewards/margins": 0.8833667096761895,
|
|
"rewards/rejected": -0.29768831184111444,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 18.81517791748047,
|
|
"kl": 16.840726852416992,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -32924400.65615142,
|
|
"logits/rejected": -34926947.071207434,
|
|
"logps/chosen": -472.1737480283912,
|
|
"logps/rejected": -356.75503095975233,
|
|
"loss": 0.5562,
|
|
"loss/base_kto": 3.1783366203308105,
|
|
"metrics/advantage_var": 459.222900390625,
|
|
"regularization/lipschitz_norm": 1313.1341552734375,
|
|
"regularization/w1": 1.2711137533187866,
|
|
"rewards/chosen": 0.6009337127396737,
|
|
"rewards/margins": 0.8580611734201835,
|
|
"rewards/rejected": -0.25712746068050985,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 16.062894821166992,
|
|
"kl": 17.200666427612305,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34852742.20970266,
|
|
"logits/rejected": -34403831.21372855,
|
|
"logps/chosen": -460.07453051643193,
|
|
"logps/rejected": -383.13472601404055,
|
|
"loss": 0.5532,
|
|
"loss/base_kto": 3.1713249683380127,
|
|
"metrics/advantage_var": 438.6906433105469,
|
|
"regularization/lipschitz_norm": 1295.8245849609375,
|
|
"regularization/w1": 1.254358172416687,
|
|
"rewards/chosen": 0.5537259336182022,
|
|
"rewards/margins": 0.8524414149522993,
|
|
"rewards/rejected": -0.2987154813340971,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 21.8310489654541,
|
|
"kl": 16.158554077148438,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34827287.16742081,
|
|
"logits/rejected": -35782742.30145867,
|
|
"logps/chosen": -472.59483408748116,
|
|
"logps/rejected": -368.5598662884927,
|
|
"loss": 0.5546,
|
|
"loss/base_kto": 3.186894178390503,
|
|
"metrics/advantage_var": 418.841552734375,
|
|
"regularization/lipschitz_norm": 1291.564697265625,
|
|
"regularization/w1": 1.250234603881836,
|
|
"rewards/chosen": 0.6041385426240808,
|
|
"rewards/margins": 0.83494204622011,
|
|
"rewards/rejected": -0.23080350359602916,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 16.86695671081543,
|
|
"kl": 15.236518859863281,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -33886632.81733746,
|
|
"logits/rejected": -35228839.97476341,
|
|
"logps/chosen": -491.03521671826627,
|
|
"logps/rejected": -351.9504879731861,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.1471123695373535,
|
|
"metrics/advantage_var": 472.02471923828125,
|
|
"regularization/lipschitz_norm": 1371.6966552734375,
|
|
"regularization/w1": 1.327802300453186,
|
|
"rewards/chosen": 0.6444367683339783,
|
|
"rewards/margins": 0.892264244089351,
|
|
"rewards/rejected": -0.24782747575537264,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 17.506450653076172,
|
|
"kl": 14.43799114227295,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35159176.735905044,
|
|
"logits/rejected": -34792326.33663366,
|
|
"logps/chosen": -468.8777355341246,
|
|
"logps/rejected": -390.7791357260726,
|
|
"loss": 0.5632,
|
|
"loss/base_kto": 3.252534866333008,
|
|
"metrics/advantage_var": 408.421142578125,
|
|
"regularization/lipschitz_norm": 1294.4359130859375,
|
|
"regularization/w1": 1.2530139684677124,
|
|
"rewards/chosen": 0.5193015984328635,
|
|
"rewards/margins": 0.7791759485998138,
|
|
"rewards/rejected": -0.25987435016695026,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 13.688880920410156,
|
|
"kl": 18.5648193359375,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34833847.32903226,
|
|
"logits/rejected": -35717064.14545455,
|
|
"logps/chosen": -480.2965221774194,
|
|
"logps/rejected": -393.7698390151515,
|
|
"loss": 0.5591,
|
|
"loss/base_kto": 3.169818639755249,
|
|
"metrics/advantage_var": 437.3623046875,
|
|
"regularization/lipschitz_norm": 1345.8072509765625,
|
|
"regularization/w1": 1.3027414083480835,
|
|
"rewards/chosen": 0.5905017483618952,
|
|
"rewards/margins": 0.8669500848764549,
|
|
"rewards/rejected": -0.27644833651455963,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 15.005108833312988,
|
|
"kl": 16.943044662475586,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34711051.095975235,
|
|
"logits/rejected": -35973746.67507886,
|
|
"logps/chosen": -468.4868421052632,
|
|
"logps/rejected": -383.1545248422713,
|
|
"loss": 0.5551,
|
|
"loss/base_kto": 3.2320587635040283,
|
|
"metrics/advantage_var": 435.31915283203125,
|
|
"regularization/lipschitz_norm": 1248.82568359375,
|
|
"regularization/w1": 1.2088632583618164,
|
|
"rewards/chosen": 0.5310895228902622,
|
|
"rewards/margins": 0.8049011038971998,
|
|
"rewards/rejected": -0.2738115810069376,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 14.439000129699707,
|
|
"kl": 14.053105354309082,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -32798904.44720497,
|
|
"logits/rejected": -35138939.974842764,
|
|
"logps/chosen": -503.3350155279503,
|
|
"logps/rejected": -353.41548742138366,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.2001991271972656,
|
|
"metrics/advantage_var": 444.739501953125,
|
|
"regularization/lipschitz_norm": 1357.4036865234375,
|
|
"regularization/w1": 1.3139667510986328,
|
|
"rewards/chosen": 0.5621507064155911,
|
|
"rewards/margins": 0.8263029453693793,
|
|
"rewards/rejected": -0.2641522389537883,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 13.330938339233398,
|
|
"kl": 16.989704132080078,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -33937784.23003195,
|
|
"logits/rejected": -36362856.905198775,
|
|
"logps/chosen": -491.1370307507987,
|
|
"logps/rejected": -378.1333142201835,
|
|
"loss": 0.5489,
|
|
"loss/base_kto": 3.195096254348755,
|
|
"metrics/advantage_var": 410.2955017089844,
|
|
"regularization/lipschitz_norm": 1235.5635986328125,
|
|
"regularization/w1": 1.1960254907608032,
|
|
"rewards/chosen": 0.5536118772464057,
|
|
"rewards/margins": 0.8186561100027608,
|
|
"rewards/rejected": -0.26504423275635514,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 17.127193450927734,
|
|
"kl": 16.70460319519043,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -33892532.89589905,
|
|
"logits/rejected": -35187651.76470588,
|
|
"logps/chosen": -489.6055796529968,
|
|
"logps/rejected": -370.78700657894734,
|
|
"loss": 0.5568,
|
|
"loss/base_kto": 3.193934202194214,
|
|
"metrics/advantage_var": 392.8123474121094,
|
|
"regularization/lipschitz_norm": 1302.4627685546875,
|
|
"regularization/w1": 1.2607840299606323,
|
|
"rewards/chosen": 0.544373810103263,
|
|
"rewards/margins": 0.8132462866869269,
|
|
"rewards/rejected": -0.26887247658366387,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 21.913311004638672,
|
|
"kl": 18.2757511138916,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -33734282.132471725,
|
|
"logits/rejected": -35014287.29803328,
|
|
"logps/chosen": -497.5305432148627,
|
|
"logps/rejected": -370.5315336611195,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.165632724761963,
|
|
"metrics/advantage_var": 480.1612243652344,
|
|
"regularization/lipschitz_norm": 1329.922119140625,
|
|
"regularization/w1": 1.2873646020889282,
|
|
"rewards/chosen": 0.6263517462956886,
|
|
"rewards/margins": 0.8615022436629588,
|
|
"rewards/rejected": -0.23515049736727023,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 14.240015029907227,
|
|
"kl": 16.6707763671875,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34648265.44262295,
|
|
"logits/rejected": -36758738.91343284,
|
|
"logps/chosen": -491.2049692622951,
|
|
"logps/rejected": -378.88027052238806,
|
|
"loss": 0.5525,
|
|
"loss/base_kto": 3.080538511276245,
|
|
"metrics/advantage_var": 493.7511901855469,
|
|
"regularization/lipschitz_norm": 1384.0885009765625,
|
|
"regularization/w1": 1.3397976160049438,
|
|
"rewards/chosen": 0.6625063036308914,
|
|
"rewards/margins": 0.9604570482597976,
|
|
"rewards/rejected": -0.29795074462890625,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 15.475432395935059,
|
|
"kl": 15.486937522888184,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34422012.28985507,
|
|
"logits/rejected": -36248782.6646434,
|
|
"logps/chosen": -468.6735104669887,
|
|
"logps/rejected": -371.3877560698027,
|
|
"loss": 0.5559,
|
|
"loss/base_kto": 3.142998456954956,
|
|
"metrics/advantage_var": 464.4955139160156,
|
|
"regularization/lipschitz_norm": 1346.9864501953125,
|
|
"regularization/w1": 1.3038829565048218,
|
|
"rewards/chosen": 0.6192004807329409,
|
|
"rewards/margins": 0.9063298611862793,
|
|
"rewards/rejected": -0.2871293804533384,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 19.727224349975586,
|
|
"kl": 17.00147819519043,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34866807.27733756,
|
|
"logits/rejected": -35465831.34668721,
|
|
"logps/chosen": -467.4975237717908,
|
|
"logps/rejected": -385.7292469183359,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.1698315143585205,
|
|
"metrics/advantage_var": 459.7564392089844,
|
|
"regularization/lipschitz_norm": 1310.61181640625,
|
|
"regularization/w1": 1.268672227859497,
|
|
"rewards/chosen": 0.5610522770466027,
|
|
"rewards/margins": 0.8665984777444913,
|
|
"rewards/rejected": -0.30554620069788857,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 16.78679656982422,
|
|
"kl": 15.156946182250977,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34897055.584415585,
|
|
"logits/rejected": -36429401.44578313,
|
|
"logps/chosen": -493.0818790584416,
|
|
"logps/rejected": -400.7662838855422,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.1686346530914307,
|
|
"metrics/advantage_var": 461.27764892578125,
|
|
"regularization/lipschitz_norm": 1390.584228515625,
|
|
"regularization/w1": 1.346085548400879,
|
|
"rewards/chosen": 0.5905988123509791,
|
|
"rewards/margins": 0.8635635656512897,
|
|
"rewards/rejected": -0.2729647533003106,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 17.594505310058594,
|
|
"kl": 14.354560852050781,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33768229.717868336,
|
|
"logits/rejected": -35560929.69470405,
|
|
"logps/chosen": -464.0438871473354,
|
|
"logps/rejected": -364.80821651090343,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.218501329421997,
|
|
"metrics/advantage_var": 408.1050720214844,
|
|
"regularization/lipschitz_norm": 1306.2288818359375,
|
|
"regularization/w1": 1.2644294500350952,
|
|
"rewards/chosen": 0.5158534513371865,
|
|
"rewards/margins": 0.7974783136753387,
|
|
"rewards/rejected": -0.28162486233815226,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 20.96672248840332,
|
|
"kl": 16.356908798217773,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34671509.63100775,
|
|
"logits/rejected": -35817662.28661417,
|
|
"logps/chosen": -493.24646317829456,
|
|
"logps/rejected": -393.57445866141734,
|
|
"loss": 0.5647,
|
|
"loss/base_kto": 3.20023512840271,
|
|
"metrics/advantage_var": 475.9938659667969,
|
|
"regularization/lipschitz_norm": 1360.5635986328125,
|
|
"regularization/w1": 1.3170255422592163,
|
|
"rewards/chosen": 0.5620458321977956,
|
|
"rewards/margins": 0.8555970113777907,
|
|
"rewards/rejected": -0.29355117917999507,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 12.60053825378418,
|
|
"kl": 14.49432373046875,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33889937.28679818,
|
|
"logits/rejected": -33668282.33172303,
|
|
"logps/chosen": -469.68114567526555,
|
|
"logps/rejected": -342.706672705314,
|
|
"loss": 0.5459,
|
|
"loss/base_kto": 3.164848804473877,
|
|
"metrics/advantage_var": 418.7054748535156,
|
|
"regularization/lipschitz_norm": 1241.8707275390625,
|
|
"regularization/w1": 1.2021307945251465,
|
|
"rewards/chosen": 0.5740103598610584,
|
|
"rewards/margins": 0.8767580525159245,
|
|
"rewards/rejected": -0.3027476926548661,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_loss": 0.5861780454045956,
|
|
"train_runtime": 11125.761,
|
|
"train_samples_per_second": 13.322,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.975174191323546e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|