1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 10.285362243652344,
|
|
"kl": 55.3999137878418,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -38240443.33643411,
|
|
"logits/rejected": -38302824.818897635,
|
|
"logps/chosen": -489.996996124031,
|
|
"logps/rejected": -354.4371062992126,
|
|
"loss": 0.4594,
|
|
"loss/base_kto": 3.675299882888794,
|
|
"metrics/advantage_var": 640.5584716796875,
|
|
"rewards/chosen": 1.0600410875423933,
|
|
"rewards/margins": 0.3018039366442634,
|
|
"rewards/rejected": 0.7582371508981299,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 11.506349563598633,
|
|
"kl": 35.54319381713867,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37366320.5382263,
|
|
"logits/rejected": -38880592.971246004,
|
|
"logps/chosen": -479.0196387614679,
|
|
"logps/rejected": -371.96016373801916,
|
|
"loss": 0.4534,
|
|
"loss/base_kto": 3.6272010803222656,
|
|
"metrics/advantage_var": 710.2990112304688,
|
|
"rewards/chosen": 0.7506291100738245,
|
|
"rewards/margins": 0.343996421594142,
|
|
"rewards/rejected": 0.4066326884796825,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 10.1532564163208,
|
|
"kl": 31.856287002563477,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -38610258.201834865,
|
|
"logits/rejected": -38919020.77955271,
|
|
"logps/chosen": -463.1452599388379,
|
|
"logps/rejected": -351.8382587859425,
|
|
"loss": 0.4361,
|
|
"loss/base_kto": 3.488476514816284,
|
|
"metrics/advantage_var": 788.6171875,
|
|
"rewards/chosen": 0.7459707930911936,
|
|
"rewards/margins": 0.5374012816532443,
|
|
"rewards/rejected": 0.20856951143794927,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 10.552718162536621,
|
|
"kl": 19.224849700927734,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -38283614.11195159,
|
|
"logits/rejected": -39037983.43134087,
|
|
"logps/chosen": -497.19265317700456,
|
|
"logps/rejected": -383.6700323101777,
|
|
"loss": 0.4227,
|
|
"loss/base_kto": 3.381673574447632,
|
|
"metrics/advantage_var": 1146.7421875,
|
|
"rewards/chosen": 0.671803946213715,
|
|
"rewards/margins": 0.732286359608871,
|
|
"rewards/rejected": -0.060482413395156,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 9.475604057312012,
|
|
"kl": 11.81312084197998,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -37433490.28571428,
|
|
"logits/rejected": -39362005.46461538,
|
|
"logps/chosen": -490.66299603174605,
|
|
"logps/rejected": -365.20064903846156,
|
|
"loss": 0.4078,
|
|
"loss/base_kto": 3.2620036602020264,
|
|
"metrics/advantage_var": 2193.870849609375,
|
|
"rewards/chosen": 0.6134866381448413,
|
|
"rewards/margins": 1.0385611949357068,
|
|
"rewards/rejected": -0.42507455679086537,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 8.439735412597656,
|
|
"kl": 32.87488555908203,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -38353533.08649469,
|
|
"logits/rejected": -39914458.074074075,
|
|
"logps/chosen": -471.7189871016692,
|
|
"logps/rejected": -354.6473429951691,
|
|
"loss": 0.3855,
|
|
"loss/base_kto": 3.0840797424316406,
|
|
"metrics/advantage_var": 3620.40234375,
|
|
"rewards/chosen": 1.2449965990729324,
|
|
"rewards/margins": 1.3967639550732092,
|
|
"rewards/rejected": -0.15176735600027677,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 8.383479118347168,
|
|
"kl": 38.29127502441406,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -38660961.716088325,
|
|
"logits/rejected": -40106234.45201238,
|
|
"logps/chosen": -478.3365536277602,
|
|
"logps/rejected": -386.499661377709,
|
|
"loss": 0.3895,
|
|
"loss/base_kto": 3.115814208984375,
|
|
"metrics/advantage_var": 3755.92041015625,
|
|
"rewards/chosen": 1.317629456144026,
|
|
"rewards/margins": 1.2437299360730967,
|
|
"rewards/rejected": 0.07389952007092927,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 10.654783248901367,
|
|
"kl": 5.33766508102417,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -38697368.25041186,
|
|
"logits/rejected": -39911624.84398217,
|
|
"logps/chosen": -488.94491350906094,
|
|
"logps/rejected": -395.10261887072807,
|
|
"loss": 0.3803,
|
|
"loss/base_kto": 3.0426743030548096,
|
|
"metrics/advantage_var": 4218.814453125,
|
|
"rewards/chosen": 0.3239868013234272,
|
|
"rewards/margins": 1.5602282255665552,
|
|
"rewards/rejected": -1.2362414242431279,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 8.015681266784668,
|
|
"kl": 8.5147066116333,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -42394393.1936508,
|
|
"logits/rejected": -42333899.22461539,
|
|
"logps/chosen": -500.61165674603177,
|
|
"logps/rejected": -386.1232451923077,
|
|
"loss": 0.3698,
|
|
"loss/base_kto": 2.958132266998291,
|
|
"metrics/advantage_var": 4260.21044921875,
|
|
"rewards/chosen": 0.66495361328125,
|
|
"rewards/margins": 1.7300279822716345,
|
|
"rewards/rejected": -1.0650743689903845,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 9.380935668945312,
|
|
"kl": 22.67376136779785,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -43375171.368421055,
|
|
"logits/rejected": -43511302.0952381,
|
|
"logps/chosen": -475.55556126644734,
|
|
"logps/rejected": -396.3759300595238,
|
|
"loss": 0.3757,
|
|
"loss/base_kto": 3.005749225616455,
|
|
"metrics/advantage_var": 4565.32275390625,
|
|
"rewards/chosen": 1.1464886916311163,
|
|
"rewards/margins": 1.739021394485818,
|
|
"rewards/rejected": -0.5925327028547015,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 7.990581035614014,
|
|
"kl": 12.28122329711914,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -42476874.531645566,
|
|
"logits/rejected": -42830000.98765432,
|
|
"logps/chosen": -489.24673655063293,
|
|
"logps/rejected": -400.7535927854938,
|
|
"loss": 0.3564,
|
|
"loss/base_kto": 2.850947856903076,
|
|
"metrics/advantage_var": 5003.28662109375,
|
|
"rewards/chosen": 0.750718225406695,
|
|
"rewards/margins": 2.134706499875011,
|
|
"rewards/rejected": -1.383988274468316,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 8.215432167053223,
|
|
"kl": 21.210721969604492,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -43546450.33918129,
|
|
"logits/rejected": -43897532.99328859,
|
|
"logps/chosen": -491.60517178362574,
|
|
"logps/rejected": -358.4549863674497,
|
|
"loss": 0.356,
|
|
"loss/base_kto": 2.847736358642578,
|
|
"metrics/advantage_var": 4862.79296875,
|
|
"rewards/chosen": 1.5007320649442617,
|
|
"rewards/margins": 1.9320024850955306,
|
|
"rewards/rejected": -0.4312704201512689,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 9.131887435913086,
|
|
"kl": 28.84025001525879,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -42420382.02469136,
|
|
"logits/rejected": -42786744.70886076,
|
|
"logps/chosen": -473.4367283950617,
|
|
"logps/rejected": -355.517948971519,
|
|
"loss": 0.3616,
|
|
"loss/base_kto": 2.892714738845825,
|
|
"metrics/advantage_var": 5250.83642578125,
|
|
"rewards/chosen": 1.442228434998312,
|
|
"rewards/margins": 1.906059696741785,
|
|
"rewards/rejected": -0.4638312617434731,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 7.937675476074219,
|
|
"kl": 13.750167846679688,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -43773237.037520394,
|
|
"logits/rejected": -44890299.298350826,
|
|
"logps/chosen": -481.702691680261,
|
|
"logps/rejected": -380.5896973388306,
|
|
"loss": 0.3431,
|
|
"loss/base_kto": 2.744920015335083,
|
|
"metrics/advantage_var": 6587.87646484375,
|
|
"rewards/chosen": 0.8538975676743474,
|
|
"rewards/margins": 2.445918817791476,
|
|
"rewards/rejected": -1.592021250117129,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 8.017827033996582,
|
|
"kl": 20.4591121673584,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -42772393.6,
|
|
"logits/rejected": -42926860.8,
|
|
"logps/chosen": -481.9798828125,
|
|
"logps/rejected": -387.5751220703125,
|
|
"loss": 0.3476,
|
|
"loss/base_kto": 2.780568838119507,
|
|
"metrics/advantage_var": 6826.62744140625,
|
|
"rewards/chosen": 1.5604660034179687,
|
|
"rewards/margins": 2.356216478347778,
|
|
"rewards/rejected": -0.7957504749298095,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 8.48287296295166,
|
|
"kl": 21.597509384155273,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -43267334.603945374,
|
|
"logits/rejected": -42670350.42834138,
|
|
"logps/chosen": -445.9710735963581,
|
|
"logps/rejected": -382.9682467793881,
|
|
"loss": 0.361,
|
|
"loss/base_kto": 2.8881566524505615,
|
|
"metrics/advantage_var": 6283.49365234375,
|
|
"rewards/chosen": 1.3021855216829477,
|
|
"rewards/margins": 2.0412563955884426,
|
|
"rewards/rejected": -0.7390708739054952,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 8.588059425354004,
|
|
"kl": 57.5013313293457,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -43813313.794392526,
|
|
"logits/rejected": -43601355.03448276,
|
|
"logps/chosen": -454.7376362928349,
|
|
"logps/rejected": -382.6814753134796,
|
|
"loss": 0.3646,
|
|
"loss/base_kto": 2.9167778491973877,
|
|
"metrics/advantage_var": 6674.80029296875,
|
|
"rewards/chosen": 2.1397389444606696,
|
|
"rewards/margins": 1.9807793733637051,
|
|
"rewards/rejected": 0.1589595710969644,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 7.111739635467529,
|
|
"kl": 25.617679595947266,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -45081461.44945568,
|
|
"logits/rejected": -44390255.321821034,
|
|
"logps/chosen": -477.66524105754274,
|
|
"logps/rejected": -389.9218995290424,
|
|
"loss": 0.3426,
|
|
"loss/base_kto": 2.7409420013427734,
|
|
"metrics/advantage_var": 7744.74853515625,
|
|
"rewards/chosen": 1.7191458267520412,
|
|
"rewards/margins": 2.679072078653042,
|
|
"rewards/rejected": -0.9599262519010008,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 7.782184600830078,
|
|
"kl": 30.644330978393555,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -45203308.8,
|
|
"logits/rejected": -45315228.8,
|
|
"logps/chosen": -485.4841796875,
|
|
"logps/rejected": -381.0125,
|
|
"loss": 0.3284,
|
|
"loss/base_kto": 2.627208709716797,
|
|
"metrics/advantage_var": 8336.6318359375,
|
|
"rewards/chosen": 1.670359230041504,
|
|
"rewards/margins": 2.859906005859375,
|
|
"rewards/rejected": -1.189546775817871,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 8.94594669342041,
|
|
"kl": 24.091541290283203,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -45122213.187800966,
|
|
"logits/rejected": -45430444.22526636,
|
|
"logps/chosen": -477.6580557784912,
|
|
"logps/rejected": -384.92660768645356,
|
|
"loss": 0.3297,
|
|
"loss/base_kto": 2.63732647895813,
|
|
"metrics/advantage_var": 9143.4013671875,
|
|
"rewards/chosen": 1.8824368839661918,
|
|
"rewards/margins": 2.9564070341516944,
|
|
"rewards/rejected": -1.0739701501855023,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 8.263615608215332,
|
|
"kl": 21.048532485961914,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -44807026.93030794,
|
|
"logits/rejected": -44261839.34841629,
|
|
"logps/chosen": -470.25040518638576,
|
|
"logps/rejected": -388.39844928355956,
|
|
"loss": 0.3381,
|
|
"loss/base_kto": 2.7047669887542725,
|
|
"metrics/advantage_var": 9148.7412109375,
|
|
"rewards/chosen": 1.416009096117808,
|
|
"rewards/margins": 2.946486720611115,
|
|
"rewards/rejected": -1.530477624493307,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 8.365622520446777,
|
|
"kl": 15.443939208984375,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -45616454.258675076,
|
|
"logits/rejected": -45114244.359133124,
|
|
"logps/chosen": -482.53677050473186,
|
|
"logps/rejected": -377.2045762383901,
|
|
"loss": 0.3365,
|
|
"loss/base_kto": 2.6917216777801514,
|
|
"metrics/advantage_var": 10364.0166015625,
|
|
"rewards/chosen": 1.4138483956033123,
|
|
"rewards/margins": 3.064291174482956,
|
|
"rewards/rejected": -1.6504427788796439,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 6.976726055145264,
|
|
"kl": 23.841480255126953,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -46631704.66445183,
|
|
"logits/rejected": -46827486.772861354,
|
|
"logps/chosen": -462.5598006644518,
|
|
"logps/rejected": -386.4936854719764,
|
|
"loss": 0.3282,
|
|
"loss/base_kto": 2.625518798828125,
|
|
"metrics/advantage_var": 10974.43359375,
|
|
"rewards/chosen": 1.5608189985205565,
|
|
"rewards/margins": 3.1252206307338675,
|
|
"rewards/rejected": -1.5644016322133112,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 10.27958869934082,
|
|
"kl": 17.574188232421875,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -45412389.52061069,
|
|
"logits/rejected": -45046031.9744,
|
|
"logps/chosen": -442.57266221374044,
|
|
"logps/rejected": -382.1891,
|
|
"loss": 0.3453,
|
|
"loss/base_kto": 2.76213002204895,
|
|
"metrics/advantage_var": 10283.8876953125,
|
|
"rewards/chosen": 1.488247331226145,
|
|
"rewards/margins": 2.860261296069895,
|
|
"rewards/rejected": -1.37201396484375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 8.24674129486084,
|
|
"kl": 53.127315521240234,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -45054310.79689922,
|
|
"logits/rejected": -43142458.45669291,
|
|
"logps/chosen": -451.6226744186047,
|
|
"logps/rejected": -376.9277066929134,
|
|
"loss": 0.3293,
|
|
"loss/base_kto": 2.6341941356658936,
|
|
"metrics/advantage_var": 9086.931640625,
|
|
"rewards/chosen": 2.6122694858284885,
|
|
"rewards/margins": 2.905609444920516,
|
|
"rewards/rejected": -0.29333995909202754,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.033597946166992,
|
|
"kl": 27.368396759033203,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -43520045.438985735,
|
|
"logits/rejected": -42915855.77812018,
|
|
"logps/chosen": -431.1351030110935,
|
|
"logps/rejected": -346.852561633282,
|
|
"loss": 0.3128,
|
|
"loss/base_kto": 2.5020527839660645,
|
|
"metrics/advantage_var": 10318.7060546875,
|
|
"rewards/chosen": 2.0742268751238115,
|
|
"rewards/margins": 3.4516921687847324,
|
|
"rewards/rejected": -1.3774652936609206,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 10.496723175048828,
|
|
"kl": 10.868855476379395,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -44145195.279106855,
|
|
"logits/rejected": -44725798.419601835,
|
|
"logps/chosen": -456.7164074960128,
|
|
"logps/rejected": -405.54881316998467,
|
|
"loss": 0.3472,
|
|
"loss/base_kto": 2.777850866317749,
|
|
"metrics/advantage_var": 12616.919921875,
|
|
"rewards/chosen": 1.2103571306195176,
|
|
"rewards/margins": 3.257504685773614,
|
|
"rewards/rejected": -2.0471475551540963,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 8.59062385559082,
|
|
"kl": 29.14273452758789,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -45075778.79939668,
|
|
"logits/rejected": -45115972.875202596,
|
|
"logps/chosen": -472.31447963800906,
|
|
"logps/rejected": -381.77162682333875,
|
|
"loss": 0.3205,
|
|
"loss/base_kto": 2.5638864040374756,
|
|
"metrics/advantage_var": 11787.837890625,
|
|
"rewards/chosen": 2.3244678618141967,
|
|
"rewards/margins": 3.2861555897315387,
|
|
"rewards/rejected": -0.961687727917342,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 9.160472869873047,
|
|
"kl": 26.70770263671875,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -47760614.12971342,
|
|
"logits/rejected": -46645672.86871961,
|
|
"logps/chosen": -488.71747737556564,
|
|
"logps/rejected": -395.1577694489465,
|
|
"loss": 0.3214,
|
|
"loss/base_kto": 2.5714452266693115,
|
|
"metrics/advantage_var": 11503.3125,
|
|
"rewards/chosen": 2.2000402850443064,
|
|
"rewards/margins": 3.423707126869671,
|
|
"rewards/rejected": -1.2236668418253647,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 7.438222408294678,
|
|
"kl": 29.855695724487305,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -45878961.10691824,
|
|
"logits/rejected": -46041781.26708075,
|
|
"logps/chosen": -475.91204795597486,
|
|
"logps/rejected": -368.5074970885093,
|
|
"loss": 0.3212,
|
|
"loss/base_kto": 2.5699808597564697,
|
|
"metrics/advantage_var": 12049.52734375,
|
|
"rewards/chosen": 2.0778695352422365,
|
|
"rewards/margins": 3.3896634608415184,
|
|
"rewards/rejected": -1.3117939255992819,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 7.2168684005737305,
|
|
"kl": 26.392969131469727,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -44772896.820512824,
|
|
"logits/rejected": -45504362.146341465,
|
|
"logps/chosen": -445.9444611378205,
|
|
"logps/rejected": -397.6354087271341,
|
|
"loss": 0.3171,
|
|
"loss/base_kto": 2.536456823348999,
|
|
"metrics/advantage_var": 13829.486328125,
|
|
"rewards/chosen": 2.100123870067107,
|
|
"rewards/margins": 3.6253483231325605,
|
|
"rewards/rejected": -1.5252244530654535,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 7.897971153259277,
|
|
"kl": 23.46337127685547,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -46756372.157480314,
|
|
"logits/rejected": -46982729.823255815,
|
|
"logps/chosen": -465.72391732283467,
|
|
"logps/rejected": -399.53880813953486,
|
|
"loss": 0.318,
|
|
"loss/base_kto": 2.543635606765747,
|
|
"metrics/advantage_var": 12641.2958984375,
|
|
"rewards/chosen": 1.9364203986220472,
|
|
"rewards/margins": 3.441243973900632,
|
|
"rewards/rejected": -1.5048235752785852,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 9.333341598510742,
|
|
"kl": 34.5018424987793,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -48553082.384266265,
|
|
"logits/rejected": -47805041.31825525,
|
|
"logps/chosen": -457.4876134644478,
|
|
"logps/rejected": -392.6214155896607,
|
|
"loss": 0.3238,
|
|
"loss/base_kto": 2.5903139114379883,
|
|
"metrics/advantage_var": 14256.0498046875,
|
|
"rewards/chosen": 2.465614584318268,
|
|
"rewards/margins": 3.4557863465166223,
|
|
"rewards/rejected": -0.9901717621983542,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 8.291895866394043,
|
|
"kl": 21.646602630615234,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -48295671.960330576,
|
|
"logits/rejected": -47792886.518518515,
|
|
"logps/chosen": -446.1569214876033,
|
|
"logps/rejected": -388.03953703703706,
|
|
"loss": 0.3251,
|
|
"loss/base_kto": 2.6008059978485107,
|
|
"metrics/advantage_var": 13585.8935546875,
|
|
"rewards/chosen": 1.8920416209323347,
|
|
"rewards/margins": 3.5260764516615017,
|
|
"rewards/rejected": -1.6340348307291668,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 8.178174018859863,
|
|
"kl": 15.580222129821777,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -48345981.443932414,
|
|
"logits/rejected": -48517605.13831478,
|
|
"logps/chosen": -470.8646313364055,
|
|
"logps/rejected": -403.8453895071542,
|
|
"loss": 0.3235,
|
|
"loss/base_kto": 2.5876247882843018,
|
|
"metrics/advantage_var": 14429.3095703125,
|
|
"rewards/chosen": 1.8095410606278801,
|
|
"rewards/margins": 3.8868899729018267,
|
|
"rewards/rejected": -2.077348912273947,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 10.443662643432617,
|
|
"kl": 35.15016555786133,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -48543886.222222224,
|
|
"logits/rejected": -48038218.531645566,
|
|
"logps/chosen": -485.367862654321,
|
|
"logps/rejected": -395.15822784810126,
|
|
"loss": 0.3011,
|
|
"loss/base_kto": 2.4088985919952393,
|
|
"metrics/advantage_var": 14492.658203125,
|
|
"rewards/chosen": 2.147234692985629,
|
|
"rewards/margins": 3.97073647304892,
|
|
"rewards/rejected": -1.8235017800632911,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 8.647013664245605,
|
|
"kl": 25.72066307067871,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -46683429.50318471,
|
|
"logits/rejected": -48587132.073619634,
|
|
"logps/chosen": -455.54488455414014,
|
|
"logps/rejected": -378.162768404908,
|
|
"loss": 0.3145,
|
|
"loss/base_kto": 2.516106128692627,
|
|
"metrics/advantage_var": 15056.3798828125,
|
|
"rewards/chosen": 1.9133522373855494,
|
|
"rewards/margins": 3.77387735690961,
|
|
"rewards/rejected": -1.8605251195240606,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 7.549548149108887,
|
|
"kl": 24.89582061767578,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -47521797.97084548,
|
|
"logits/rejected": -47149724.875420876,
|
|
"logps/chosen": -462.1880466472303,
|
|
"logps/rejected": -416.2795664983165,
|
|
"loss": 0.3123,
|
|
"loss/base_kto": 2.4984090328216553,
|
|
"metrics/advantage_var": 13964.5234375,
|
|
"rewards/chosen": 2.1415812767629374,
|
|
"rewards/margins": 3.6034990383633163,
|
|
"rewards/rejected": -1.461917761600379,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 7.165398120880127,
|
|
"kl": 36.23041915893555,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -47457207.53230769,
|
|
"logits/rejected": -47688159.38853503,
|
|
"logps/chosen": -456.6162019230769,
|
|
"logps/rejected": -383.27644804936307,
|
|
"loss": 0.2998,
|
|
"loss/base_kto": 2.398082733154297,
|
|
"metrics/advantage_var": 14451.869140625,
|
|
"rewards/chosen": 2.6437039888822116,
|
|
"rewards/margins": 4.025765989715711,
|
|
"rewards/rejected": -1.3820620008334992,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 6.993330001831055,
|
|
"kl": 6.0000762939453125,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -47126860.63694268,
|
|
"logits/rejected": -47067569.47239264,
|
|
"logps/chosen": -451.8484773089172,
|
|
"logps/rejected": -389.6282112730061,
|
|
"loss": 0.2586,
|
|
"loss/base_kto": 2.0687718391418457,
|
|
"metrics/advantage_var": 17359.888671875,
|
|
"rewards/chosen": 1.9123997779408837,
|
|
"rewards/margins": 5.285741344155416,
|
|
"rewards/rejected": -3.373341566214532,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 5.417101860046387,
|
|
"kl": 31.433074951171875,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -48107728.05079365,
|
|
"logits/rejected": -49156763.96307693,
|
|
"logps/chosen": -457.90739087301586,
|
|
"logps/rejected": -417.5091346153846,
|
|
"loss": 0.2757,
|
|
"loss/base_kto": 2.2058160305023193,
|
|
"metrics/advantage_var": 15574.1142578125,
|
|
"rewards/chosen": 2.6718709309895834,
|
|
"rewards/margins": 4.651981733273237,
|
|
"rewards/rejected": -1.9801108022836538,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 8.132774353027344,
|
|
"kl": 20.201974868774414,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -47990535.95031056,
|
|
"logits/rejected": -48448924.17610063,
|
|
"logps/chosen": -463.0153823757764,
|
|
"logps/rejected": -422.75388168238993,
|
|
"loss": 0.2484,
|
|
"loss/base_kto": 1.9872734546661377,
|
|
"metrics/advantage_var": 17828.435546875,
|
|
"rewards/chosen": 2.6961753323951863,
|
|
"rewards/margins": 5.303709680985005,
|
|
"rewards/rejected": -2.607534348589819,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 8.2401123046875,
|
|
"kl": 15.7005033493042,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -49548721.29318542,
|
|
"logits/rejected": -48194883.45146379,
|
|
"logps/chosen": -487.02595087163235,
|
|
"logps/rejected": -386.0352224576271,
|
|
"loss": 0.2666,
|
|
"loss/base_kto": 2.1330721378326416,
|
|
"metrics/advantage_var": 15226.6318359375,
|
|
"rewards/chosen": 2.371564426815075,
|
|
"rewards/margins": 4.769108688118523,
|
|
"rewards/rejected": -2.3975442613034477,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 8.672164916992188,
|
|
"kl": 20.882816314697266,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -47332732.252696455,
|
|
"logits/rejected": -48399553.11568938,
|
|
"logps/chosen": -466.80975539291217,
|
|
"logps/rejected": -404.24197702060223,
|
|
"loss": 0.2544,
|
|
"loss/base_kto": 2.0349247455596924,
|
|
"metrics/advantage_var": 17013.724609375,
|
|
"rewards/chosen": 2.4880128457843798,
|
|
"rewards/margins": 5.045798145965144,
|
|
"rewards/rejected": -2.5577853001807647,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 8.386229515075684,
|
|
"kl": 22.4251766204834,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -49451777.97226502,
|
|
"logits/rejected": -49354158.047543585,
|
|
"logps/chosen": -458.80017334360554,
|
|
"logps/rejected": -399.8250297147385,
|
|
"loss": 0.2621,
|
|
"loss/base_kto": 2.097113609313965,
|
|
"metrics/advantage_var": 14544.2685546875,
|
|
"rewards/chosen": 2.6462496388674883,
|
|
"rewards/margins": 4.811208610861348,
|
|
"rewards/rejected": -2.164958971993859,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 8.636312484741211,
|
|
"kl": 12.601666450500488,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -47768676.32970451,
|
|
"logits/rejected": -48685194.24803767,
|
|
"logps/chosen": -451.256949844479,
|
|
"logps/rejected": -393.63358516483515,
|
|
"loss": 0.2516,
|
|
"loss/base_kto": 2.012883424758911,
|
|
"metrics/advantage_var": 14977.6279296875,
|
|
"rewards/chosen": 2.213305626093507,
|
|
"rewards/margins": 4.917228586503633,
|
|
"rewards/rejected": -2.7039229604101256,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 5.238379001617432,
|
|
"kl": 29.21479606628418,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -49693457.75038521,
|
|
"logits/rejected": -50187937.47068146,
|
|
"logps/chosen": -493.2758089368259,
|
|
"logps/rejected": -401.07621830427894,
|
|
"loss": 0.247,
|
|
"loss/base_kto": 1.9758790731430054,
|
|
"metrics/advantage_var": 16148.375,
|
|
"rewards/chosen": 2.7205349726743067,
|
|
"rewards/margins": 5.184224568182429,
|
|
"rewards/rejected": -2.463689595508122,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 6.7903971672058105,
|
|
"kl": 33.20451736450195,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -48231641.972434916,
|
|
"logits/rejected": -48072583.14513557,
|
|
"logps/chosen": -481.6626148545176,
|
|
"logps/rejected": -389.2785087719298,
|
|
"loss": 0.2603,
|
|
"loss/base_kto": 2.0826306343078613,
|
|
"metrics/advantage_var": 14499.822265625,
|
|
"rewards/chosen": 3.0388022827335375,
|
|
"rewards/margins": 4.795185956539857,
|
|
"rewards/rejected": -1.7563836738063199,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 7.804295539855957,
|
|
"kl": 19.160192489624023,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -48358886.4,
|
|
"logits/rejected": -49291187.2,
|
|
"logps/chosen": -440.519384765625,
|
|
"logps/rejected": -394.4533203125,
|
|
"loss": 0.2548,
|
|
"loss/base_kto": 2.0385074615478516,
|
|
"metrics/advantage_var": 15006.9560546875,
|
|
"rewards/chosen": 2.5941253662109376,
|
|
"rewards/margins": 4.971536636352539,
|
|
"rewards/rejected": -2.3774112701416015,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 8.665066719055176,
|
|
"kl": 12.234046936035156,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -49109730.077922076,
|
|
"logits/rejected": -49753285.39759036,
|
|
"logps/chosen": -456.95200892857144,
|
|
"logps/rejected": -397.8301016566265,
|
|
"loss": 0.2594,
|
|
"loss/base_kto": 2.075338363647461,
|
|
"metrics/advantage_var": 16453.171875,
|
|
"rewards/chosen": 1.7339770824878247,
|
|
"rewards/margins": 4.999917185674007,
|
|
"rewards/rejected": -3.2659401031861823,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 6.741236209869385,
|
|
"kl": 29.949003219604492,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -47405793.54340836,
|
|
"logits/rejected": -48602127.56231003,
|
|
"logps/chosen": -454.48939911575565,
|
|
"logps/rejected": -386.7006791413374,
|
|
"loss": 0.2577,
|
|
"loss/base_kto": 2.061556100845337,
|
|
"metrics/advantage_var": 15960.3935546875,
|
|
"rewards/chosen": 2.663867423005426,
|
|
"rewards/margins": 4.971707973204894,
|
|
"rewards/rejected": -2.307840550199468,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 7.356171607971191,
|
|
"kl": 18.42108726501465,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -50573719.68895801,
|
|
"logits/rejected": -50023354.87598116,
|
|
"logps/chosen": -475.37942262830484,
|
|
"logps/rejected": -419.8053375196232,
|
|
"loss": 0.2653,
|
|
"loss/base_kto": 2.122389554977417,
|
|
"metrics/advantage_var": 14337.359375,
|
|
"rewards/chosen": 2.1847764841319983,
|
|
"rewards/margins": 4.641081444517595,
|
|
"rewards/rejected": -2.4563049603855966,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 9.710753440856934,
|
|
"kl": 20.63035011291504,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -48821714.030959755,
|
|
"logits/rejected": -49922997.70347003,
|
|
"logps/chosen": -465.2204914860681,
|
|
"logps/rejected": -399.9880224763407,
|
|
"loss": 0.2516,
|
|
"loss/base_kto": 2.0126187801361084,
|
|
"metrics/advantage_var": 15542.337890625,
|
|
"rewards/chosen": 2.6994681815982973,
|
|
"rewards/margins": 5.189337885980198,
|
|
"rewards/rejected": -2.4898697043819005,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 7.381426811218262,
|
|
"kl": 11.87850570678711,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -49439426.09672387,
|
|
"logits/rejected": -48244963.55555555,
|
|
"logps/chosen": -457.3438475039002,
|
|
"logps/rejected": -399.84497261345854,
|
|
"loss": 0.2693,
|
|
"loss/base_kto": 2.154747247695923,
|
|
"metrics/advantage_var": 14240.484375,
|
|
"rewards/chosen": 2.155618700333951,
|
|
"rewards/margins": 4.671614878138627,
|
|
"rewards/rejected": -2.5159961778046753,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 6.18265962600708,
|
|
"kl": 10.194323539733887,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -47570332.42266463,
|
|
"logits/rejected": -49176585.79904306,
|
|
"logps/chosen": -438.3728943338438,
|
|
"logps/rejected": -400.97306120414675,
|
|
"loss": 0.2297,
|
|
"loss/base_kto": 1.8375297784805298,
|
|
"metrics/advantage_var": 15763.3896484375,
|
|
"rewards/chosen": 2.536791954799962,
|
|
"rewards/margins": 5.334581216662004,
|
|
"rewards/rejected": -2.7977892618620417,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 8.46866226196289,
|
|
"kl": 24.690845489501953,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -49655544.68571428,
|
|
"logits/rejected": -49987552.49230769,
|
|
"logps/chosen": -452.3748015873016,
|
|
"logps/rejected": -388.4841586538462,
|
|
"loss": 0.2663,
|
|
"loss/base_kto": 2.130300998687744,
|
|
"metrics/advantage_var": 14631.166015625,
|
|
"rewards/chosen": 2.4113482762896825,
|
|
"rewards/margins": 4.7256937916142014,
|
|
"rewards/rejected": -2.3143455153245194,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 6.674213886260986,
|
|
"kl": 17.886266708374023,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -47965006.79938744,
|
|
"logits/rejected": -47721463.83413078,
|
|
"logps/chosen": -452.15141653905056,
|
|
"logps/rejected": -397.80699262360446,
|
|
"loss": 0.2471,
|
|
"loss/base_kto": 1.9766285419464111,
|
|
"metrics/advantage_var": 15228.6953125,
|
|
"rewards/chosen": 2.2878199251651035,
|
|
"rewards/margins": 4.977289404082567,
|
|
"rewards/rejected": -2.6894694789174642,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 8.288200378417969,
|
|
"kl": 24.316864013671875,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -50055032.75471698,
|
|
"logits/rejected": -49824163.77639752,
|
|
"logps/chosen": -465.1016607704403,
|
|
"logps/rejected": -398.7861752717391,
|
|
"loss": 0.2521,
|
|
"loss/base_kto": 2.0171926021575928,
|
|
"metrics/advantage_var": 15025.9482421875,
|
|
"rewards/chosen": 2.60492154487274,
|
|
"rewards/margins": 5.016617700491917,
|
|
"rewards/rejected": -2.411696155619177,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 9.282503128051758,
|
|
"kl": 13.939154624938965,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -50474677.67741936,
|
|
"logits/rejected": -50641575.56363636,
|
|
"logps/chosen": -481.20604838709676,
|
|
"logps/rejected": -408.43645833333335,
|
|
"loss": 0.25,
|
|
"loss/base_kto": 2.0002853870391846,
|
|
"metrics/advantage_var": 15123.5830078125,
|
|
"rewards/chosen": 2.093593277469758,
|
|
"rewards/margins": 4.947470859441823,
|
|
"rewards/rejected": -2.8538775819720645,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 9.465474128723145,
|
|
"kl": 16.497163772583008,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -48454600.434108526,
|
|
"logits/rejected": -49707682.06614173,
|
|
"logps/chosen": -461.271511627907,
|
|
"logps/rejected": -380.18929625984254,
|
|
"loss": 0.2406,
|
|
"loss/base_kto": 1.9244734048843384,
|
|
"metrics/advantage_var": 15232.501953125,
|
|
"rewards/chosen": 2.4654355544452518,
|
|
"rewards/margins": 5.215662970476256,
|
|
"rewards/rejected": -2.750227416031004,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 9.055628776550293,
|
|
"kl": 23.978981018066406,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -49989815.30864198,
|
|
"logits/rejected": -50560006.48101266,
|
|
"logps/chosen": -478.30314429012344,
|
|
"logps/rejected": -392.64883306962025,
|
|
"loss": 0.2452,
|
|
"loss/base_kto": 1.9618843793869019,
|
|
"metrics/advantage_var": 15591.2568359375,
|
|
"rewards/chosen": 2.683606183087384,
|
|
"rewards/margins": 5.173647795157202,
|
|
"rewards/rejected": -2.490041612069818,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 6.475921154022217,
|
|
"kl": 13.34227180480957,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -48551734.04006163,
|
|
"logits/rejected": -48786677.045958795,
|
|
"logps/chosen": -440.99017719568565,
|
|
"logps/rejected": -377.6189332408875,
|
|
"loss": 0.2512,
|
|
"loss/base_kto": 2.009838581085205,
|
|
"metrics/advantage_var": 15655.4658203125,
|
|
"rewards/chosen": 2.48008617524557,
|
|
"rewards/margins": 4.9992728813049006,
|
|
"rewards/rejected": -2.5191867060593305,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 7.807542324066162,
|
|
"kl": 19.20454216003418,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -51067883.78947368,
|
|
"logits/rejected": -51769593.9047619,
|
|
"logps/chosen": -488.40409128289474,
|
|
"logps/rejected": -407.9049479166667,
|
|
"loss": 0.2542,
|
|
"loss/base_kto": 2.033954620361328,
|
|
"metrics/advantage_var": 16857.92578125,
|
|
"rewards/chosen": 1.9239341334292763,
|
|
"rewards/margins": 5.132258384150073,
|
|
"rewards/rejected": -3.2083242507207963,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 6.524475574493408,
|
|
"kl": 23.61891746520996,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -48980962.02157165,
|
|
"logits/rejected": -49016809.28050713,
|
|
"logps/chosen": -468.9685092449923,
|
|
"logps/rejected": -386.245418977813,
|
|
"loss": 0.2513,
|
|
"loss/base_kto": 2.0107860565185547,
|
|
"metrics/advantage_var": 15216.9609375,
|
|
"rewards/chosen": 2.5923028366958785,
|
|
"rewards/margins": 4.926098205839599,
|
|
"rewards/rejected": -2.3337953691437203,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 5.242307186126709,
|
|
"kl": 22.227195739746094,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -49624793.61978362,
|
|
"logits/rejected": -49335724.68878357,
|
|
"logps/chosen": -451.85896445131374,
|
|
"logps/rejected": -386.2475562796209,
|
|
"loss": 0.2486,
|
|
"loss/base_kto": 1.9891036748886108,
|
|
"metrics/advantage_var": 16150.3232421875,
|
|
"rewards/chosen": 2.3942293759659967,
|
|
"rewards/margins": 5.14253813324779,
|
|
"rewards/rejected": -2.748308757281793,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 9.66505241394043,
|
|
"kl": 22.66203498840332,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -50443175.5431918,
|
|
"logits/rejected": -49279524.87772194,
|
|
"logps/chosen": -451.52379209370423,
|
|
"logps/rejected": -381.40991415410383,
|
|
"loss": 0.264,
|
|
"loss/base_kto": 2.1119725704193115,
|
|
"metrics/advantage_var": 14104.1640625,
|
|
"rewards/chosen": 2.5510915194912154,
|
|
"rewards/margins": 4.746612377361295,
|
|
"rewards/rejected": -2.1955208578700796,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 6.532532215118408,
|
|
"kl": 16.468366622924805,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -51127444.06697108,
|
|
"logits/rejected": -52672495.56340289,
|
|
"logps/chosen": -449.2677891933029,
|
|
"logps/rejected": -387.2645465489567,
|
|
"loss": 0.2703,
|
|
"loss/base_kto": 2.162468433380127,
|
|
"metrics/advantage_var": 14352.1005859375,
|
|
"rewards/chosen": 2.2623242707738775,
|
|
"rewards/margins": 4.768650373678171,
|
|
"rewards/rejected": -2.5063261029042936,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 9.158368110656738,
|
|
"kl": 26.074420928955078,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -50660605.70746268,
|
|
"logits/rejected": -50919967.89508197,
|
|
"logps/chosen": -441.9116604477612,
|
|
"logps/rejected": -411.3995901639344,
|
|
"loss": 0.2705,
|
|
"loss/base_kto": 2.164259672164917,
|
|
"metrics/advantage_var": 14123.228515625,
|
|
"rewards/chosen": 2.9296955165578358,
|
|
"rewards/margins": 4.729554635410295,
|
|
"rewards/rejected": -1.799859118852459,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 9.339201927185059,
|
|
"kl": 27.37833023071289,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -50341196.63694268,
|
|
"logits/rejected": -50491489.37423313,
|
|
"logps/chosen": -432.54578025477707,
|
|
"logps/rejected": -408.8282208588957,
|
|
"loss": 0.2559,
|
|
"loss/base_kto": 2.046827793121338,
|
|
"metrics/advantage_var": 13350.6435546875,
|
|
"rewards/chosen": 2.771653290766819,
|
|
"rewards/margins": 4.713148285135109,
|
|
"rewards/rejected": -1.94149499436829,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 8.989336967468262,
|
|
"kl": 17.84269905090332,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -49385063.646879755,
|
|
"logits/rejected": -49007443.415730335,
|
|
"logps/chosen": -481.3707191780822,
|
|
"logps/rejected": -401.7123294542536,
|
|
"loss": 0.2502,
|
|
"loss/base_kto": 2.0016610622406006,
|
|
"metrics/advantage_var": 14103.3125,
|
|
"rewards/chosen": 2.322153773663432,
|
|
"rewards/margins": 5.013345273964395,
|
|
"rewards/rejected": -2.691191500300963,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 8.873215675354004,
|
|
"kl": 25.032546997070312,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -48653897.3784219,
|
|
"logits/rejected": -48986276.710166916,
|
|
"logps/chosen": -460.5298913043478,
|
|
"logps/rejected": -387.5011380880121,
|
|
"loss": 0.2557,
|
|
"loss/base_kto": 2.0458295345306396,
|
|
"metrics/advantage_var": 13571.0654296875,
|
|
"rewards/chosen": 2.3807917547302737,
|
|
"rewards/margins": 4.739925207404781,
|
|
"rewards/rejected": -2.3591334526745067,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 7.878697872161865,
|
|
"kl": 22.722684860229492,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -50874733.147286825,
|
|
"logits/rejected": -52412786.89763779,
|
|
"logps/chosen": -482.9199612403101,
|
|
"logps/rejected": -405.6564960629921,
|
|
"loss": 0.2612,
|
|
"loss/base_kto": 2.0892159938812256,
|
|
"metrics/advantage_var": 13676.0380859375,
|
|
"rewards/chosen": 2.6010393955910853,
|
|
"rewards/margins": 4.692092891223467,
|
|
"rewards/rejected": -2.091053495632382,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 8.507803916931152,
|
|
"kl": 23.1352596282959,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -48004160.726094,
|
|
"logits/rejected": -49818035.54751131,
|
|
"logps/chosen": -450.5213735818476,
|
|
"logps/rejected": -391.52747926093514,
|
|
"loss": 0.235,
|
|
"loss/base_kto": 1.880078911781311,
|
|
"metrics/advantage_var": 14898.5185546875,
|
|
"rewards/chosen": 2.3386291800673624,
|
|
"rewards/margins": 5.1856568508111405,
|
|
"rewards/rejected": -2.8470276707437785,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 7.470147609710693,
|
|
"kl": 15.821776390075684,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -49612119.48346457,
|
|
"logits/rejected": -50831232.99224806,
|
|
"logps/chosen": -438.76028543307086,
|
|
"logps/rejected": -388.28682170542635,
|
|
"loss": 0.2585,
|
|
"loss/base_kto": 2.067831516265869,
|
|
"metrics/advantage_var": 14473.900390625,
|
|
"rewards/chosen": 2.060779096948819,
|
|
"rewards/margins": 4.792124103610641,
|
|
"rewards/rejected": -2.731345006661822,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 6.032121658325195,
|
|
"kl": 17.546789169311523,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -52378246.056782335,
|
|
"logits/rejected": -52407920.54489164,
|
|
"logps/chosen": -488.7177641955836,
|
|
"logps/rejected": -397.67942143962847,
|
|
"loss": 0.2422,
|
|
"loss/base_kto": 1.9377278089523315,
|
|
"metrics/advantage_var": 15465.8251953125,
|
|
"rewards/chosen": 2.5849694092567033,
|
|
"rewards/margins": 5.338190289251382,
|
|
"rewards/rejected": -2.7532208799946787,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 7.5287394523620605,
|
|
"kl": 25.733800888061523,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -51796815.335463256,
|
|
"logits/rejected": -51933249.76146789,
|
|
"logps/chosen": -471.7799520766773,
|
|
"logps/rejected": -385.5217173165138,
|
|
"loss": 0.2477,
|
|
"loss/base_kto": 1.9814552068710327,
|
|
"metrics/advantage_var": 15103.8818359375,
|
|
"rewards/chosen": 2.7011432099266175,
|
|
"rewards/margins": 5.033878884023521,
|
|
"rewards/rejected": -2.3327356740969036,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 9.235956192016602,
|
|
"kl": 29.76057243347168,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -52228933.06623586,
|
|
"logits/rejected": -51011735.81845688,
|
|
"logps/chosen": -465.15690630048465,
|
|
"logps/rejected": -424.50902987897126,
|
|
"loss": 0.2564,
|
|
"loss/base_kto": 2.0513575077056885,
|
|
"metrics/advantage_var": 15029.6220703125,
|
|
"rewards/chosen": 2.412079794148829,
|
|
"rewards/margins": 4.9222398853931555,
|
|
"rewards/rejected": -2.5101600912443267,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 9.331817626953125,
|
|
"kl": 25.071556091308594,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -51467012.24736048,
|
|
"logits/rejected": -50623096.71544716,
|
|
"logps/chosen": -449.86156674208144,
|
|
"logps/rejected": -428.4603658536585,
|
|
"loss": 0.2127,
|
|
"loss/base_kto": 1.7012656927108765,
|
|
"metrics/advantage_var": 15838.7451171875,
|
|
"rewards/chosen": 2.864487223675528,
|
|
"rewards/margins": 5.68939889637624,
|
|
"rewards/rejected": -2.8249116727007113,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 7.488211154937744,
|
|
"kl": 15.822104454040527,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -50769451.14149444,
|
|
"logits/rejected": -50692316.21505377,
|
|
"logps/chosen": -452.8102146263911,
|
|
"logps/rejected": -400.2412634408602,
|
|
"loss": 0.2097,
|
|
"loss/base_kto": 1.6774075031280518,
|
|
"metrics/advantage_var": 17759.546875,
|
|
"rewards/chosen": 2.733765036578398,
|
|
"rewards/margins": 6.010643381792492,
|
|
"rewards/rejected": -3.2768783452140937,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 7.268414497375488,
|
|
"kl": 9.772688865661621,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -49696237.77346278,
|
|
"logits/rejected": -50958045.196374625,
|
|
"logps/chosen": -463.1455299352751,
|
|
"logps/rejected": -399.58780211480365,
|
|
"loss": 0.218,
|
|
"loss/base_kto": 1.7441755533218384,
|
|
"metrics/advantage_var": 17008.37109375,
|
|
"rewards/chosen": 2.337260323434972,
|
|
"rewards/margins": 6.028836955562992,
|
|
"rewards/rejected": -3.691576632128021,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 9.88840103149414,
|
|
"kl": 22.12735939025879,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -50274113.64102564,
|
|
"logits/rejected": -51329461.07317073,
|
|
"logps/chosen": -448.53205128205127,
|
|
"logps/rejected": -396.9389291158537,
|
|
"loss": 0.2082,
|
|
"loss/base_kto": 1.6654434204101562,
|
|
"metrics/advantage_var": 18491.703125,
|
|
"rewards/chosen": 2.5738243689903846,
|
|
"rewards/margins": 5.829478308586421,
|
|
"rewards/rejected": -3.2556539395960367,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 6.961165428161621,
|
|
"kl": 18.75223731994629,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -50454448.49689441,
|
|
"logits/rejected": -50288182.74213836,
|
|
"logps/chosen": -487.41605201863354,
|
|
"logps/rejected": -402.01046580188677,
|
|
"loss": 0.2131,
|
|
"loss/base_kto": 1.7046397924423218,
|
|
"metrics/advantage_var": 15868.8974609375,
|
|
"rewards/chosen": 2.810227672505823,
|
|
"rewards/margins": 5.734512793439883,
|
|
"rewards/rejected": -2.9242851209340603,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 6.136966228485107,
|
|
"kl": 18.14278221130371,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -49533700.24736048,
|
|
"logits/rejected": -50127515.17666127,
|
|
"logps/chosen": -445.97134238310707,
|
|
"logps/rejected": -397.2188006482982,
|
|
"loss": 0.2009,
|
|
"loss/base_kto": 1.6072006225585938,
|
|
"metrics/advantage_var": 16655.65625,
|
|
"rewards/chosen": 2.8560848092241704,
|
|
"rewards/margins": 5.881715420105957,
|
|
"rewards/rejected": -3.025630610881787,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 8.093600273132324,
|
|
"kl": 17.36097526550293,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -50249372.24413145,
|
|
"logits/rejected": -51871648.14976599,
|
|
"logps/chosen": -471.8325508607199,
|
|
"logps/rejected": -434.346236349454,
|
|
"loss": 0.2028,
|
|
"loss/base_kto": 1.6226786375045776,
|
|
"metrics/advantage_var": 17302.814453125,
|
|
"rewards/chosen": 2.768492792693662,
|
|
"rewards/margins": 5.937769085229738,
|
|
"rewards/rejected": -3.1692762925360762,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.161694526672363,
|
|
"kl": 14.578579902648926,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -51127046.74728682,
|
|
"logits/rejected": -51333645.707086615,
|
|
"logps/chosen": -435.92863372093024,
|
|
"logps/rejected": -429.8619586614173,
|
|
"loss": 0.2108,
|
|
"loss/base_kto": 1.6867679357528687,
|
|
"metrics/advantage_var": 15839.5849609375,
|
|
"rewards/chosen": 2.5460812590843025,
|
|
"rewards/margins": 5.6935579599947355,
|
|
"rewards/rejected": -3.147476700910433,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 7.36198091506958,
|
|
"kl": 19.158613204956055,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -50113851.57232705,
|
|
"logits/rejected": -51104122.43478261,
|
|
"logps/chosen": -465.19536163522014,
|
|
"logps/rejected": -397.4027319487578,
|
|
"loss": 0.2135,
|
|
"loss/base_kto": 1.7078889608383179,
|
|
"metrics/advantage_var": 16287.8193359375,
|
|
"rewards/chosen": 2.5575148864362225,
|
|
"rewards/margins": 5.749604654366152,
|
|
"rewards/rejected": -3.19208976792993,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 9.570615768432617,
|
|
"kl": 17.231595993041992,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -50655309.40472441,
|
|
"logits/rejected": -49515077.06046512,
|
|
"logps/chosen": -457.63287401574803,
|
|
"logps/rejected": -404.8307897286822,
|
|
"loss": 0.2221,
|
|
"loss/base_kto": 1.7766475677490234,
|
|
"metrics/advantage_var": 15297.1845703125,
|
|
"rewards/chosen": 2.469990311269685,
|
|
"rewards/margins": 5.49773869123577,
|
|
"rewards/rejected": -3.027748379966085,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 9.824966430664062,
|
|
"kl": 19.26784324645996,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -52412467.28012519,
|
|
"logits/rejected": -50879607.81279251,
|
|
"logps/chosen": -460.15307120500785,
|
|
"logps/rejected": -395.80703978159124,
|
|
"loss": 0.23,
|
|
"loss/base_kto": 1.8403421640396118,
|
|
"metrics/advantage_var": 14083.1220703125,
|
|
"rewards/chosen": 2.4090841708235526,
|
|
"rewards/margins": 5.085994333167546,
|
|
"rewards/rejected": -2.6769101623439937,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 6.393725395202637,
|
|
"kl": 15.844639778137207,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -50632716.43095599,
|
|
"logits/rejected": -51273401.507246375,
|
|
"logps/chosen": -446.82312215478,
|
|
"logps/rejected": -389.93422906602257,
|
|
"loss": 0.2031,
|
|
"loss/base_kto": 1.6246970891952515,
|
|
"metrics/advantage_var": 16746.81640625,
|
|
"rewards/chosen": 2.7439853975009485,
|
|
"rewards/margins": 5.871646962768661,
|
|
"rewards/rejected": -3.1276615652677133,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 7.77438497543335,
|
|
"kl": 28.328439712524414,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -52250043.568438,
|
|
"logits/rejected": -53629083.38694993,
|
|
"logps/chosen": -470.0669283413849,
|
|
"logps/rejected": -411.1176498482549,
|
|
"loss": 0.1992,
|
|
"loss/base_kto": 1.5935217142105103,
|
|
"metrics/advantage_var": 15910.0224609375,
|
|
"rewards/chosen": 3.0569956314160627,
|
|
"rewards/margins": 5.844844467246867,
|
|
"rewards/rejected": -2.7878488358308045,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 6.337928771972656,
|
|
"kl": 19.673072814941406,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -51213865.86163522,
|
|
"logits/rejected": -52022189.31677019,
|
|
"logps/chosen": -450.1392492138365,
|
|
"logps/rejected": -394.90855493012424,
|
|
"loss": 0.2168,
|
|
"loss/base_kto": 1.7344608306884766,
|
|
"metrics/advantage_var": 16663.849609375,
|
|
"rewards/chosen": 2.961632302722091,
|
|
"rewards/margins": 5.595088894215686,
|
|
"rewards/rejected": -2.633456591493595,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 8.079915046691895,
|
|
"kl": 16.748624801635742,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -53084767.64835165,
|
|
"logits/rejected": -52579020.640746504,
|
|
"logps/chosen": -467.8896683673469,
|
|
"logps/rejected": -415.1285964230171,
|
|
"loss": 0.221,
|
|
"loss/base_kto": 1.768080711364746,
|
|
"metrics/advantage_var": 16247.9970703125,
|
|
"rewards/chosen": 2.7143727157329276,
|
|
"rewards/margins": 5.659321852469903,
|
|
"rewards/rejected": -2.9449491367369753,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 5.926120281219482,
|
|
"kl": 17.201818466186523,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -51085710.222222224,
|
|
"logits/rejected": -52309610.93670886,
|
|
"logps/chosen": -451.0401234567901,
|
|
"logps/rejected": -372.73615506329116,
|
|
"loss": 0.2095,
|
|
"loss/base_kto": 1.6759108304977417,
|
|
"metrics/advantage_var": 14099.5068359375,
|
|
"rewards/chosen": 2.6364719720534335,
|
|
"rewards/margins": 5.612774879490143,
|
|
"rewards/rejected": -2.976302907436709,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 7.158931255340576,
|
|
"kl": 15.858980178833008,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -50755445.20481928,
|
|
"logits/rejected": -51616528.62337662,
|
|
"logps/chosen": -461.50009412650604,
|
|
"logps/rejected": -415.9528206168831,
|
|
"loss": 0.2032,
|
|
"loss/base_kto": 1.625422477722168,
|
|
"metrics/advantage_var": 15971.3046875,
|
|
"rewards/chosen": 2.809034967996988,
|
|
"rewards/margins": 5.888142871811922,
|
|
"rewards/rejected": -3.079107903814935,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 7.33803653717041,
|
|
"kl": 19.720277786254883,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -50146619.81308411,
|
|
"logits/rejected": -50380286.39498433,
|
|
"logps/chosen": -454.36365848909657,
|
|
"logps/rejected": -386.84357856583074,
|
|
"loss": 0.2157,
|
|
"loss/base_kto": 1.7253481149673462,
|
|
"metrics/advantage_var": 15806.0859375,
|
|
"rewards/chosen": 2.822266195422021,
|
|
"rewards/margins": 5.662788602786343,
|
|
"rewards/rejected": -2.8405224073643223,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 6.7824177742004395,
|
|
"kl": 18.44866371154785,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -51437188.12903226,
|
|
"logits/rejected": -52641382.4,
|
|
"logps/chosen": -472.04944556451613,
|
|
"logps/rejected": -402.6609848484849,
|
|
"loss": 0.2058,
|
|
"loss/base_kto": 1.6467617750167847,
|
|
"metrics/advantage_var": 17072.724609375,
|
|
"rewards/chosen": 2.6735591765372986,
|
|
"rewards/margins": 5.771285709444495,
|
|
"rewards/rejected": -3.097726532907197,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 7.855797290802002,
|
|
"kl": 16.122150421142578,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -51113090.556067586,
|
|
"logits/rejected": -51917897.2591415,
|
|
"logps/chosen": -470.721678187404,
|
|
"logps/rejected": -417.7419515103339,
|
|
"loss": 0.2039,
|
|
"loss/base_kto": 1.6308526992797852,
|
|
"metrics/advantage_var": 17782.169921875,
|
|
"rewards/chosen": 2.5794075820852536,
|
|
"rewards/margins": 6.061200668830087,
|
|
"rewards/rejected": -3.4817930867448332,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.747197151184082,
|
|
"kl": 14.113003730773926,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -50281886.3627907,
|
|
"logits/rejected": -52991419.46456693,
|
|
"logps/chosen": -472.15610465116276,
|
|
"logps/rejected": -378.7300196850394,
|
|
"loss": 0.2021,
|
|
"loss/base_kto": 1.6164880990982056,
|
|
"metrics/advantage_var": 14748.0537109375,
|
|
"rewards/chosen": 2.6983644470687986,
|
|
"rewards/margins": 5.782121406037795,
|
|
"rewards/rejected": -3.083756958968996,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 6.209463596343994,
|
|
"kl": 15.16028118133545,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -49942296.4006462,
|
|
"logits/rejected": -51573172.86535552,
|
|
"logps/chosen": -440.99828352180936,
|
|
"logps/rejected": -403.07531202723146,
|
|
"loss": 0.2099,
|
|
"loss/base_kto": 1.679508090019226,
|
|
"metrics/advantage_var": 16923.134765625,
|
|
"rewards/chosen": 2.595790289908118,
|
|
"rewards/margins": 6.006971856663035,
|
|
"rewards/rejected": -3.411181566754917,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 7.013144016265869,
|
|
"kl": 14.801623344421387,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -50693691.96396396,
|
|
"logits/rejected": -51218171.830618896,
|
|
"logps/chosen": -453.25079767267266,
|
|
"logps/rejected": -415.81479030944627,
|
|
"loss": 0.2103,
|
|
"loss/base_kto": 1.6823161840438843,
|
|
"metrics/advantage_var": 15928.6845703125,
|
|
"rewards/chosen": 2.736321893182245,
|
|
"rewards/margins": 5.762679140103051,
|
|
"rewards/rejected": -3.026357246920806,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 7.630949020385742,
|
|
"kl": 18.435169219970703,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -52484132.870703764,
|
|
"logits/rejected": -51555151.9760837,
|
|
"logps/chosen": -465.0326309328969,
|
|
"logps/rejected": -394.2265508221226,
|
|
"loss": 0.199,
|
|
"loss/base_kto": 1.5920383930206299,
|
|
"metrics/advantage_var": 14698.828125,
|
|
"rewards/chosen": 2.884322096204992,
|
|
"rewards/margins": 5.721907636657159,
|
|
"rewards/rejected": -2.8375855404521673,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 10.548248291015625,
|
|
"kl": 19.13457489013672,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -52158762.342943855,
|
|
"logits/rejected": -51546728.70853462,
|
|
"logps/chosen": -462.69850151745067,
|
|
"logps/rejected": -411.6543377616747,
|
|
"loss": 0.2165,
|
|
"loss/base_kto": 1.7320812940597534,
|
|
"metrics/advantage_var": 14811.025390625,
|
|
"rewards/chosen": 2.6523830199639606,
|
|
"rewards/margins": 5.49644517399979,
|
|
"rewards/rejected": -2.8440621540358295,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 6.244058609008789,
|
|
"kl": 26.8873233795166,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -50528268.70077519,
|
|
"logits/rejected": -51241775.97480315,
|
|
"logps/chosen": -460.00116279069766,
|
|
"logps/rejected": -399.81500984251966,
|
|
"loss": 0.2054,
|
|
"loss/base_kto": 1.643135666847229,
|
|
"metrics/advantage_var": 14407.8173828125,
|
|
"rewards/chosen": 2.8557206501332364,
|
|
"rewards/margins": 5.599035041742488,
|
|
"rewards/rejected": -2.743314391609252,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 8.370528221130371,
|
|
"kl": 25.490325927734375,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -53414775.359497644,
|
|
"logits/rejected": -53532280.23639191,
|
|
"logps/chosen": -463.2737441130298,
|
|
"logps/rejected": -394.83257192846037,
|
|
"loss": 0.2222,
|
|
"loss/base_kto": 1.7775534391403198,
|
|
"metrics/advantage_var": 14866.4375,
|
|
"rewards/chosen": 2.80948139472135,
|
|
"rewards/margins": 5.5009601654979825,
|
|
"rewards/rejected": -2.691478770776633,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.29795503616333,
|
|
"kl": 20.725143432617188,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -50126091.96261682,
|
|
"logits/rejected": -52081834.131661445,
|
|
"logps/chosen": -438.7496105919003,
|
|
"logps/rejected": -399.84926038401255,
|
|
"loss": 0.2124,
|
|
"loss/base_kto": 1.6995877027511597,
|
|
"metrics/advantage_var": 14963.5888671875,
|
|
"rewards/chosen": 2.7964680989583335,
|
|
"rewards/margins": 5.438754632804336,
|
|
"rewards/rejected": -2.6422865338460033,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 8.448776245117188,
|
|
"kl": 15.388836860656738,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -50943257.52265861,
|
|
"logits/rejected": -50960695.508090615,
|
|
"logps/chosen": -452.53663141993957,
|
|
"logps/rejected": -375.4554004854369,
|
|
"loss": 0.2081,
|
|
"loss/base_kto": 1.6651920080184937,
|
|
"metrics/advantage_var": 14255.4609375,
|
|
"rewards/chosen": 3.0594740576614425,
|
|
"rewards/margins": 5.675124222318097,
|
|
"rewards/rejected": -2.6156501646566546,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 7.798056602478027,
|
|
"kl": 20.17386245727539,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -51199567.4624,
|
|
"logits/rejected": -51550445.63053435,
|
|
"logps/chosen": -455.71965,
|
|
"logps/rejected": -413.4286259541985,
|
|
"loss": 0.2098,
|
|
"loss/base_kto": 1.678598403930664,
|
|
"metrics/advantage_var": 16558.5,
|
|
"rewards/chosen": 2.8292068359375,
|
|
"rewards/margins": 5.923838537690839,
|
|
"rewards/rejected": -3.0946317017533396,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 7.845084190368652,
|
|
"kl": 20.583127975463867,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -53823648.5632,
|
|
"logits/rejected": -52538316.40916031,
|
|
"logps/chosen": -484.1464,
|
|
"logps/rejected": -412.46030534351144,
|
|
"loss": 0.2159,
|
|
"loss/base_kto": 1.7270206212997437,
|
|
"metrics/advantage_var": 16702.03515625,
|
|
"rewards/chosen": 2.6395771484375,
|
|
"rewards/margins": 5.676251915851623,
|
|
"rewards/rejected": -3.036674767414122,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 6.940150737762451,
|
|
"kl": 16.465957641601562,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -51824299.2180937,
|
|
"logits/rejected": -51926562.85627837,
|
|
"logps/chosen": -458.44476978998387,
|
|
"logps/rejected": -406.61365355521934,
|
|
"loss": 0.2052,
|
|
"loss/base_kto": 1.641516089439392,
|
|
"metrics/advantage_var": 14746.0771484375,
|
|
"rewards/chosen": 2.707992036046042,
|
|
"rewards/margins": 5.695761181694207,
|
|
"rewards/rejected": -2.9877691456481656,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 8.538128852844238,
|
|
"kl": 15.000971794128418,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -52334231.32618683,
|
|
"logits/rejected": -51295815.0430622,
|
|
"logps/chosen": -469.02340160796325,
|
|
"logps/rejected": -422.6503688197767,
|
|
"loss": 0.211,
|
|
"loss/base_kto": 1.6879322528839111,
|
|
"metrics/advantage_var": 16872.240234375,
|
|
"rewards/chosen": 2.591564698387251,
|
|
"rewards/margins": 5.641363934581429,
|
|
"rewards/rejected": -3.0497992361941786,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 8.950865745544434,
|
|
"kl": 21.08583641052246,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -51087206.48165869,
|
|
"logits/rejected": -50116241.05359878,
|
|
"logps/chosen": -446.7995414673046,
|
|
"logps/rejected": -396.5116050918836,
|
|
"loss": 0.2158,
|
|
"loss/base_kto": 1.72600257396698,
|
|
"metrics/advantage_var": 17154.9921875,
|
|
"rewards/chosen": 2.645399565141547,
|
|
"rewards/margins": 5.832831893697252,
|
|
"rewards/rejected": -3.1874323285557042,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 8.326827049255371,
|
|
"kl": 17.32566261291504,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -52515542.10909091,
|
|
"logits/rejected": -51846609.75483871,
|
|
"logps/chosen": -461.6983428030303,
|
|
"logps/rejected": -394.51542338709675,
|
|
"loss": 0.2242,
|
|
"loss/base_kto": 1.7936210632324219,
|
|
"metrics/advantage_var": 14815.369140625,
|
|
"rewards/chosen": 2.720080196496212,
|
|
"rewards/margins": 5.3184590239911715,
|
|
"rewards/rejected": -2.5983788274949595,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.218096733093262,
|
|
"kl": 15.86826229095459,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -51368914.91823899,
|
|
"logits/rejected": -50538655.00621118,
|
|
"logps/chosen": -457.04255110062894,
|
|
"logps/rejected": -402.4126067546584,
|
|
"loss": 0.2094,
|
|
"loss/base_kto": 1.6755253076553345,
|
|
"metrics/advantage_var": 14514.9814453125,
|
|
"rewards/chosen": 2.684348244337166,
|
|
"rewards/margins": 5.517094667146269,
|
|
"rewards/rejected": -2.832746422809103,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.36819314956665,
|
|
"kl": 17.891759872436523,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -51435028.86520376,
|
|
"logits/rejected": -50859017.57009346,
|
|
"logps/chosen": -462.7827194357367,
|
|
"logps/rejected": -381.15272098909657,
|
|
"loss": 0.1961,
|
|
"loss/base_kto": 1.568899154663086,
|
|
"metrics/advantage_var": 14655.2255859375,
|
|
"rewards/chosen": 2.8108076615767046,
|
|
"rewards/margins": 5.7752992710490565,
|
|
"rewards/rejected": -2.964491609472352,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 7.518880844116211,
|
|
"kl": 13.201791763305664,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -51772033.81417323,
|
|
"logits/rejected": -52071339.85736434,
|
|
"logps/chosen": -462.1588090551181,
|
|
"logps/rejected": -407.6047480620155,
|
|
"loss": 0.2082,
|
|
"loss/base_kto": 1.6658432483673096,
|
|
"metrics/advantage_var": 15404.8857421875,
|
|
"rewards/chosen": 2.8972886934055118,
|
|
"rewards/margins": 5.823128991370628,
|
|
"rewards/rejected": -2.925840297965116,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_loss": 0.27261167929374286,
|
|
"train_runtime": 11062.6242,
|
|
"train_samples_per_second": 13.398,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|