1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 10.319388389587402,
|
||
|
|
"kl": 16.727964401245117,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37122425.3480315,
|
||
|
|
"logits/rejected": -37737895.88837209,
|
||
|
|
"logps/chosen": -463.6321358267717,
|
||
|
|
"logps/rejected": -374.01157945736435,
|
||
|
|
"loss": 0.4688,
|
||
|
|
"loss/base_kto": 3.7502057552337646,
|
||
|
|
"metrics/advantage_var": 476.1357421875,
|
||
|
|
"rewards/chosen": 0.39666541392408955,
|
||
|
|
"rewards/margins": 0.25662812570188753,
|
||
|
|
"rewards/rejected": 0.14003728822220204,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 10.064275741577148,
|
||
|
|
"kl": 13.24753475189209,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -37318842.18181818,
|
||
|
|
"logits/rejected": -38839044.46275753,
|
||
|
|
"logps/chosen": -480.88530431432974,
|
||
|
|
"logps/rejected": -376.75906299524564,
|
||
|
|
"loss": 0.4448,
|
||
|
|
"loss/base_kto": 3.558764696121216,
|
||
|
|
"metrics/advantage_var": 669.9764404296875,
|
||
|
|
"rewards/chosen": 0.49594534710853233,
|
||
|
|
"rewards/margins": 0.5053853006420133,
|
||
|
|
"rewards/rejected": -0.009439953533480927,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 9.0316801071167,
|
||
|
|
"kl": 12.896925926208496,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -37219113.674418606,
|
||
|
|
"logits/rejected": -37509648.93228346,
|
||
|
|
"logps/chosen": -470.2107073643411,
|
||
|
|
"logps/rejected": -360.15996555118113,
|
||
|
|
"loss": 0.426,
|
||
|
|
"loss/base_kto": 3.4081313610076904,
|
||
|
|
"metrics/advantage_var": 1473.3863525390625,
|
||
|
|
"rewards/chosen": 0.5174255844234497,
|
||
|
|
"rewards/margins": 0.6775934647445618,
|
||
|
|
"rewards/rejected": -0.1601678803211122,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 9.75800609588623,
|
||
|
|
"kl": 22.625085830688477,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -40073294.40735069,
|
||
|
|
"logits/rejected": -40570089.543859646,
|
||
|
|
"logps/chosen": -483.6102603369066,
|
||
|
|
"logps/rejected": -376.1366128389155,
|
||
|
|
"loss": 0.42,
|
||
|
|
"loss/base_kto": 3.360121965408325,
|
||
|
|
"metrics/advantage_var": 2046.6541748046875,
|
||
|
|
"rewards/chosen": 0.9029619529455398,
|
||
|
|
"rewards/margins": 0.7994733654489663,
|
||
|
|
"rewards/rejected": 0.10348858749657347,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 10.479662895202637,
|
||
|
|
"kl": 7.0752153396606445,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -39373104.80267558,
|
||
|
|
"logits/rejected": -39785216.75073314,
|
||
|
|
"logps/chosen": -505.0452550167224,
|
||
|
|
"logps/rejected": -389.1406708211144,
|
||
|
|
"loss": 0.4144,
|
||
|
|
"loss/base_kto": 3.315232038497925,
|
||
|
|
"metrics/advantage_var": 2967.476806640625,
|
||
|
|
"rewards/chosen": 0.3449013496322377,
|
||
|
|
"rewards/margins": 1.078456133786151,
|
||
|
|
"rewards/rejected": -0.7335547841539132,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 7.468899726867676,
|
||
|
|
"kl": 29.222803115844727,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -38817840.07511737,
|
||
|
|
"logits/rejected": -39936057.510140404,
|
||
|
|
"logps/chosen": -447.14568661971833,
|
||
|
|
"logps/rejected": -359.3209584633385,
|
||
|
|
"loss": 0.4112,
|
||
|
|
"loss/base_kto": 3.2892253398895264,
|
||
|
|
"metrics/advantage_var": 3382.028076171875,
|
||
|
|
"rewards/chosen": 0.9050804938135759,
|
||
|
|
"rewards/margins": 0.9898408294174298,
|
||
|
|
"rewards/rejected": -0.08476033560385385,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 8.750174522399902,
|
||
|
|
"kl": 12.275041580200195,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -42091332.37267081,
|
||
|
|
"logits/rejected": -41663439.69811321,
|
||
|
|
"logps/chosen": -491.394264363354,
|
||
|
|
"logps/rejected": -385.76407724056605,
|
||
|
|
"loss": 0.3905,
|
||
|
|
"loss/base_kto": 3.1238603591918945,
|
||
|
|
"metrics/advantage_var": 4545.78857421875,
|
||
|
|
"rewards/chosen": 0.9180897777865392,
|
||
|
|
"rewards/margins": 1.4688283223706096,
|
||
|
|
"rewards/rejected": -0.5507385445840703,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 8.236615180969238,
|
||
|
|
"kl": 16.803390502929688,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -40913541.08868501,
|
||
|
|
"logits/rejected": -40773331.01597444,
|
||
|
|
"logps/chosen": -465.6788035168196,
|
||
|
|
"logps/rejected": -370.9947084664537,
|
||
|
|
"loss": 0.3854,
|
||
|
|
"loss/base_kto": 3.083263635635376,
|
||
|
|
"metrics/advantage_var": 4547.63671875,
|
||
|
|
"rewards/chosen": 0.8942556585376051,
|
||
|
|
"rewards/margins": 1.597708844377761,
|
||
|
|
"rewards/rejected": -0.7034531858401557,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 8.821699142456055,
|
||
|
|
"kl": 21.234865188598633,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -41670907.288343556,
|
||
|
|
"logits/rejected": -41453613.65605096,
|
||
|
|
"logps/chosen": -453.00896280674846,
|
||
|
|
"logps/rejected": -369.6968053343949,
|
||
|
|
"loss": 0.3838,
|
||
|
|
"loss/base_kto": 3.070784091949463,
|
||
|
|
"metrics/advantage_var": 4485.45849609375,
|
||
|
|
"rewards/chosen": 1.1577457357769363,
|
||
|
|
"rewards/margins": 1.5722619660746324,
|
||
|
|
"rewards/rejected": -0.4145162302976961,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 9.399519920349121,
|
||
|
|
"kl": 16.941009521484375,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -43490414.276923075,
|
||
|
|
"logits/rejected": -42209328.76190476,
|
||
|
|
"logps/chosen": -481.90163461538464,
|
||
|
|
"logps/rejected": -407.65835813492066,
|
||
|
|
"loss": 0.3737,
|
||
|
|
"loss/base_kto": 2.9893739223480225,
|
||
|
|
"metrics/advantage_var": 5094.6171875,
|
||
|
|
"rewards/chosen": 1.177627704326923,
|
||
|
|
"rewards/margins": 1.7559307719732522,
|
||
|
|
"rewards/rejected": -0.5783030676463293,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 8.513607025146484,
|
||
|
|
"kl": 19.285432815551758,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -42722276.67189953,
|
||
|
|
"logits/rejected": -41917699.58320373,
|
||
|
|
"logps/chosen": -482.56323587127156,
|
||
|
|
"logps/rejected": -386.4064930015552,
|
||
|
|
"loss": 0.3646,
|
||
|
|
"loss/base_kto": 2.9167091846466064,
|
||
|
|
"metrics/advantage_var": 6109.7998046875,
|
||
|
|
"rewards/chosen": 1.1926577102445546,
|
||
|
|
"rewards/margins": 1.9864442174313255,
|
||
|
|
"rewards/rejected": -0.793786507186771,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 11.65726375579834,
|
||
|
|
"kl": 14.581194877624512,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -44301136.36245955,
|
||
|
|
"logits/rejected": -43484528.145015106,
|
||
|
|
"logps/chosen": -471.87591019417476,
|
||
|
|
"logps/rejected": -390.4674046450151,
|
||
|
|
"loss": 0.3696,
|
||
|
|
"loss/base_kto": 2.956815719604492,
|
||
|
|
"metrics/advantage_var": 6047.51123046875,
|
||
|
|
"rewards/chosen": 0.8881642363218042,
|
||
|
|
"rewards/margins": 1.9577712362982016,
|
||
|
|
"rewards/rejected": -1.0696069999763973,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 9.075246810913086,
|
||
|
|
"kl": 13.598471641540527,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -43582794.21498372,
|
||
|
|
"logits/rejected": -43640010.95495495,
|
||
|
|
"logps/chosen": -507.21686685667754,
|
||
|
|
"logps/rejected": -383.68947072072075,
|
||
|
|
"loss": 0.3573,
|
||
|
|
"loss/base_kto": 2.8587589263916016,
|
||
|
|
"metrics/advantage_var": 6647.6220703125,
|
||
|
|
"rewards/chosen": 1.0324669257049064,
|
||
|
|
"rewards/margins": 2.329373634659955,
|
||
|
|
"rewards/rejected": -1.2969067089550488,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 7.176731586456299,
|
||
|
|
"kl": 17.565298080444336,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -44210205.4912,
|
||
|
|
"logits/rejected": -44493703.62137405,
|
||
|
|
"logps/chosen": -507.2881,
|
||
|
|
"logps/rejected": -404.58382633587786,
|
||
|
|
"loss": 0.3346,
|
||
|
|
"loss/base_kto": 2.6767992973327637,
|
||
|
|
"metrics/advantage_var": 6408.30615234375,
|
||
|
|
"rewards/chosen": 0.931298828125,
|
||
|
|
"rewards/margins": 2.5220157070014313,
|
||
|
|
"rewards/rejected": -1.5907168788764312,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 8.242661476135254,
|
||
|
|
"kl": 9.632750511169434,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -43166918.4,
|
||
|
|
"logits/rejected": -43010860.8,
|
||
|
|
"logps/chosen": -483.6794921875,
|
||
|
|
"logps/rejected": -361.163037109375,
|
||
|
|
"loss": 0.3398,
|
||
|
|
"loss/base_kto": 2.7182796001434326,
|
||
|
|
"metrics/advantage_var": 7442.8330078125,
|
||
|
|
"rewards/chosen": 1.415898895263672,
|
||
|
|
"rewards/margins": 2.555503082275391,
|
||
|
|
"rewards/rejected": -1.1396041870117188,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 6.161278247833252,
|
||
|
|
"kl": 11.036481857299805,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -43577174.89908257,
|
||
|
|
"logits/rejected": -45461203.01597444,
|
||
|
|
"logps/chosen": -480.4206804281346,
|
||
|
|
"logps/rejected": -401.44393969648564,
|
||
|
|
"loss": 0.3462,
|
||
|
|
"loss/base_kto": 2.7698185443878174,
|
||
|
|
"metrics/advantage_var": 8291.142578125,
|
||
|
|
"rewards/chosen": 0.9927509086212252,
|
||
|
|
"rewards/margins": 2.7852506824206458,
|
||
|
|
"rewards/rejected": -1.7924997737994208,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 9.249044418334961,
|
||
|
|
"kl": 28.86688804626465,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -46465939.79785605,
|
||
|
|
"logits/rejected": -45670164.82296651,
|
||
|
|
"logps/chosen": -476.4023258039816,
|
||
|
|
"logps/rejected": -384.04410885167465,
|
||
|
|
"loss": 0.3626,
|
||
|
|
"loss/base_kto": 2.9010062217712402,
|
||
|
|
"metrics/advantage_var": 7303.671875,
|
||
|
|
"rewards/chosen": 1.7468029916012635,
|
||
|
|
"rewards/margins": 2.238367095842691,
|
||
|
|
"rewards/rejected": -0.49156410424142744,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 9.388398170471191,
|
||
|
|
"kl": 25.781204223632812,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -45289124.102564104,
|
||
|
|
"logits/rejected": -45208925.65853658,
|
||
|
|
"logps/chosen": -481.48011818910254,
|
||
|
|
"logps/rejected": -395.6542492378049,
|
||
|
|
"loss": 0.3361,
|
||
|
|
"loss/base_kto": 2.6887481212615967,
|
||
|
|
"metrics/advantage_var": 8222.0947265625,
|
||
|
|
"rewards/chosen": 1.5168069692758412,
|
||
|
|
"rewards/margins": 2.7474430077071483,
|
||
|
|
"rewards/rejected": -1.230636038431307,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 9.163446426391602,
|
||
|
|
"kl": 19.06619644165039,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -44553825.940902025,
|
||
|
|
"logits/rejected": -44534547.692307696,
|
||
|
|
"logps/chosen": -471.5203635303266,
|
||
|
|
"logps/rejected": -364.72792386185245,
|
||
|
|
"loss": 0.3452,
|
||
|
|
"loss/base_kto": 2.7618725299835205,
|
||
|
|
"metrics/advantage_var": 7323.47998046875,
|
||
|
|
"rewards/chosen": 1.4123068137636081,
|
||
|
|
"rewards/margins": 2.6482218735224876,
|
||
|
|
"rewards/rejected": -1.2359150597588795,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 8.153302192687988,
|
||
|
|
"kl": 22.222400665283203,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -44122445.98153846,
|
||
|
|
"logits/rejected": -45661642.36190476,
|
||
|
|
"logps/chosen": -492.3990865384615,
|
||
|
|
"logps/rejected": -374.6800843253968,
|
||
|
|
"loss": 0.3357,
|
||
|
|
"loss/base_kto": 2.6852593421936035,
|
||
|
|
"metrics/advantage_var": 8430.4921875,
|
||
|
|
"rewards/chosen": 1.713446326622596,
|
||
|
|
"rewards/margins": 2.7226771673964056,
|
||
|
|
"rewards/rejected": -1.0092308407738095,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 7.898141860961914,
|
||
|
|
"kl": 20.997982025146484,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -46517715.75308642,
|
||
|
|
"logits/rejected": -45683141.67088608,
|
||
|
|
"logps/chosen": -496.25810185185185,
|
||
|
|
"logps/rejected": -403.4624456091772,
|
||
|
|
"loss": 0.3356,
|
||
|
|
"loss/base_kto": 2.6848278045654297,
|
||
|
|
"metrics/advantage_var": 9729.1015625,
|
||
|
|
"rewards/chosen": 1.6485524118682484,
|
||
|
|
"rewards/margins": 3.0480907658968777,
|
||
|
|
"rewards/rejected": -1.3995383540286293,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 8.437301635742188,
|
||
|
|
"kl": 27.33742332458496,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -47016659.97452229,
|
||
|
|
"logits/rejected": -46722393.521472394,
|
||
|
|
"logps/chosen": -472.54428742038215,
|
||
|
|
"logps/rejected": -388.16971817484665,
|
||
|
|
"loss": 0.3348,
|
||
|
|
"loss/base_kto": 2.678339719772339,
|
||
|
|
"metrics/advantage_var": 10187.8037109375,
|
||
|
|
"rewards/chosen": 1.7116751701209196,
|
||
|
|
"rewards/margins": 2.885346120232356,
|
||
|
|
"rewards/rejected": -1.173670950111436,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 10.420513153076172,
|
||
|
|
"kl": 29.8809814453125,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -45925395.14018691,
|
||
|
|
"logits/rejected": -45740982.169278994,
|
||
|
|
"logps/chosen": -458.6536214953271,
|
||
|
|
"logps/rejected": -377.3076998432602,
|
||
|
|
"loss": 0.3142,
|
||
|
|
"loss/base_kto": 2.513406276702881,
|
||
|
|
"metrics/advantage_var": 9882.564453125,
|
||
|
|
"rewards/chosen": 1.8956435729410046,
|
||
|
|
"rewards/margins": 3.2017841497251442,
|
||
|
|
"rewards/rejected": -1.3061405767841399,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 8.041691780090332,
|
||
|
|
"kl": 29.984716415405273,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -46290135.748427674,
|
||
|
|
"logits/rejected": -46272499.27950311,
|
||
|
|
"logps/chosen": -453.2558962264151,
|
||
|
|
"logps/rejected": -389.4093070652174,
|
||
|
|
"loss": 0.3321,
|
||
|
|
"loss/base_kto": 2.656989336013794,
|
||
|
|
"metrics/advantage_var": 9887.2763671875,
|
||
|
|
"rewards/chosen": 2.058879348466981,
|
||
|
|
"rewards/margins": 2.9575133172230466,
|
||
|
|
"rewards/rejected": -0.8986339687560656,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 8.568934440612793,
|
||
|
|
"kl": 14.765207290649414,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -46030885.57798165,
|
||
|
|
"logits/rejected": -44988641.73801917,
|
||
|
|
"logps/chosen": -455.4039564220183,
|
||
|
|
"logps/rejected": -384.76402755591056,
|
||
|
|
"loss": 0.3225,
|
||
|
|
"loss/base_kto": 2.579798936843872,
|
||
|
|
"metrics/advantage_var": 9816.142578125,
|
||
|
|
"rewards/chosen": 1.8881363708309442,
|
||
|
|
"rewards/margins": 3.233141589044303,
|
||
|
|
"rewards/rejected": -1.3450052182133587,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 6.7368597984313965,
|
||
|
|
"kl": 30.476104736328125,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -45823653.71980676,
|
||
|
|
"logits/rejected": -45456227.05918058,
|
||
|
|
"logps/chosen": -443.19444444444446,
|
||
|
|
"logps/rejected": -379.547467754173,
|
||
|
|
"loss": 0.3302,
|
||
|
|
"loss/base_kto": 2.6419026851654053,
|
||
|
|
"metrics/advantage_var": 9941.6689453125,
|
||
|
|
"rewards/chosen": 1.8815548416113124,
|
||
|
|
"rewards/margins": 3.1469455184773025,
|
||
|
|
"rewards/rejected": -1.2653906768659902,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 9.077353477478027,
|
||
|
|
"kl": 32.544036865234375,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -46674559.04903418,
|
||
|
|
"logits/rejected": -47708240.9752883,
|
||
|
|
"logps/chosen": -480.0802841753343,
|
||
|
|
"logps/rejected": -386.7947899505766,
|
||
|
|
"loss": 0.316,
|
||
|
|
"loss/base_kto": 2.5276782512664795,
|
||
|
|
"metrics/advantage_var": 10337.0224609375,
|
||
|
|
"rewards/chosen": 2.4605105259797546,
|
||
|
|
"rewards/margins": 3.265589274403097,
|
||
|
|
"rewards/rejected": -0.8050787484233423,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 6.8249897956848145,
|
||
|
|
"kl": 51.18595504760742,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -46264067.2,
|
||
|
|
"logits/rejected": -47504531.2,
|
||
|
|
"logps/chosen": -462.7865234375,
|
||
|
|
"logps/rejected": -388.075048828125,
|
||
|
|
"loss": 0.3242,
|
||
|
|
"loss/base_kto": 2.593670606613159,
|
||
|
|
"metrics/advantage_var": 9831.8037109375,
|
||
|
|
"rewards/chosen": 2.6040317535400392,
|
||
|
|
"rewards/margins": 3.0919196128845217,
|
||
|
|
"rewards/rejected": -0.4878878593444824,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 5.395700931549072,
|
||
|
|
"kl": 16.848234176635742,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -46107721.37071651,
|
||
|
|
"logits/rejected": -46889512.12539185,
|
||
|
|
"logps/chosen": -458.63721767912773,
|
||
|
|
"logps/rejected": -370.9887343260188,
|
||
|
|
"loss": 0.3291,
|
||
|
|
"loss/base_kto": 2.632402181625366,
|
||
|
|
"metrics/advantage_var": 11003.7841796875,
|
||
|
|
"rewards/chosen": 1.5970843068535825,
|
||
|
|
"rewards/margins": 3.241402412138653,
|
||
|
|
"rewards/rejected": -1.6443181052850706,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 7.256164073944092,
|
||
|
|
"kl": 40.421199798583984,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -47410145.6474259,
|
||
|
|
"logits/rejected": -48573313.40219092,
|
||
|
|
"logps/chosen": -472.3001170046802,
|
||
|
|
"logps/rejected": -365.27787558685446,
|
||
|
|
"loss": 0.3276,
|
||
|
|
"loss/base_kto": 2.620436668395996,
|
||
|
|
"metrics/advantage_var": 12412.2998046875,
|
||
|
|
"rewards/chosen": 2.108003089654836,
|
||
|
|
"rewards/margins": 2.9913788514958473,
|
||
|
|
"rewards/rejected": -0.8833757618410113,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 9.761085510253906,
|
||
|
|
"kl": 29.126300811767578,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -47805701.14374034,
|
||
|
|
"logits/rejected": -47122011.39968404,
|
||
|
|
"logps/chosen": -470.47498068006183,
|
||
|
|
"logps/rejected": -376.11349723538706,
|
||
|
|
"loss": 0.3132,
|
||
|
|
"loss/base_kto": 2.5052177906036377,
|
||
|
|
"metrics/advantage_var": 11753.0498046875,
|
||
|
|
"rewards/chosen": 2.456938235063273,
|
||
|
|
"rewards/margins": 3.7659804440075364,
|
||
|
|
"rewards/rejected": -1.3090422089442635,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 7.527853965759277,
|
||
|
|
"kl": 18.0603084564209,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -46224885.10638298,
|
||
|
|
"logits/rejected": -47408901.76205788,
|
||
|
|
"logps/chosen": -451.393047112462,
|
||
|
|
"logps/rejected": -392.36600683279744,
|
||
|
|
"loss": 0.3303,
|
||
|
|
"loss/base_kto": 2.6424295902252197,
|
||
|
|
"metrics/advantage_var": 12895.171875,
|
||
|
|
"rewards/chosen": 1.7432306631720176,
|
||
|
|
"rewards/margins": 3.2953440888653454,
|
||
|
|
"rewards/rejected": -1.552113425693328,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 7.182866096496582,
|
||
|
|
"kl": 20.800764083862305,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -47058805.152542375,
|
||
|
|
"logits/rejected": -47342600.1141046,
|
||
|
|
"logps/chosen": -462.9138097072419,
|
||
|
|
"logps/rejected": -400.24985142630743,
|
||
|
|
"loss": 0.3148,
|
||
|
|
"loss/base_kto": 2.5186355113983154,
|
||
|
|
"metrics/advantage_var": 13971.6015625,
|
||
|
|
"rewards/chosen": 2.1760500303953196,
|
||
|
|
"rewards/margins": 3.83824800793593,
|
||
|
|
"rewards/rejected": -1.6621979775406102,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 8.891425132751465,
|
||
|
|
"kl": 17.636144638061523,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -44455313.82278481,
|
||
|
|
"logits/rejected": -44785935.802469134,
|
||
|
|
"logps/chosen": -461.2084157436709,
|
||
|
|
"logps/rejected": -389.9142554012346,
|
||
|
|
"loss": 0.3401,
|
||
|
|
"loss/base_kto": 2.7205283641815186,
|
||
|
|
"metrics/advantage_var": 13725.75,
|
||
|
|
"rewards/chosen": 1.3341555052165743,
|
||
|
|
"rewards/margins": 3.3661077862736732,
|
||
|
|
"rewards/rejected": -2.031952281057099,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 7.126125812530518,
|
||
|
|
"kl": 20.660459518432617,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -45936999.52351097,
|
||
|
|
"logits/rejected": -46289495.7258567,
|
||
|
|
"logps/chosen": -450.60624020376173,
|
||
|
|
"logps/rejected": -373.2173870716511,
|
||
|
|
"loss": 0.3209,
|
||
|
|
"loss/base_kto": 2.567134141921997,
|
||
|
|
"metrics/advantage_var": 12918.2548828125,
|
||
|
|
"rewards/chosen": 1.6922259196218652,
|
||
|
|
"rewards/margins": 3.6139123000735784,
|
||
|
|
"rewards/rejected": -1.9216863804517135,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 6.310122013092041,
|
||
|
|
"kl": 27.32685661315918,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -47984717.40472441,
|
||
|
|
"logits/rejected": -47533724.37829457,
|
||
|
|
"logps/chosen": -470.71077755905515,
|
||
|
|
"logps/rejected": -380.2222625968992,
|
||
|
|
"loss": 0.3195,
|
||
|
|
"loss/base_kto": 2.5560810565948486,
|
||
|
|
"metrics/advantage_var": 14027.7880859375,
|
||
|
|
"rewards/chosen": 2.4276184562623033,
|
||
|
|
"rewards/margins": 3.788381461955133,
|
||
|
|
"rewards/rejected": -1.3607630056928295,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 10.711471557617188,
|
||
|
|
"kl": 18.596837997436523,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -47200074.45101088,
|
||
|
|
"logits/rejected": -48807614.492935635,
|
||
|
|
"logps/chosen": -459.070567651633,
|
||
|
|
"logps/rejected": -409.03264815541604,
|
||
|
|
"loss": 0.3205,
|
||
|
|
"loss/base_kto": 2.564175844192505,
|
||
|
|
"metrics/advantage_var": 17230.35546875,
|
||
|
|
"rewards/chosen": 1.8940837854174766,
|
||
|
|
"rewards/margins": 4.052348233023443,
|
||
|
|
"rewards/rejected": -2.1582644476059656,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 5.282236576080322,
|
||
|
|
"kl": 16.107288360595703,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -45939061.867528275,
|
||
|
|
"logits/rejected": -45671859.3161876,
|
||
|
|
"logps/chosen": -464.7586833602585,
|
||
|
|
"logps/rejected": -385.3533472012103,
|
||
|
|
"loss": 0.324,
|
||
|
|
"loss/base_kto": 2.5922391414642334,
|
||
|
|
"metrics/advantage_var": 14907.953125,
|
||
|
|
"rewards/chosen": 1.3830086210432653,
|
||
|
|
"rewards/margins": 3.8709718603350582,
|
||
|
|
"rewards/rejected": -2.4879632392917927,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 6.966801643371582,
|
||
|
|
"kl": 33.476539611816406,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -47285591.44358578,
|
||
|
|
"logits/rejected": -47422079.39144216,
|
||
|
|
"logps/chosen": -481.49927550231837,
|
||
|
|
"logps/rejected": -387.48085875594296,
|
||
|
|
"loss": 0.2973,
|
||
|
|
"loss/base_kto": 2.3787217140197754,
|
||
|
|
"metrics/advantage_var": 14550.7412109375,
|
||
|
|
"rewards/chosen": 2.6301737436002703,
|
||
|
|
"rewards/margins": 4.041040555023111,
|
||
|
|
"rewards/rejected": -1.4108668114228406,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 7.4706525802612305,
|
||
|
|
"kl": 26.45621681213379,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -46360979.2,
|
||
|
|
"logits/rejected": -46993430.4,
|
||
|
|
"logps/chosen": -458.044189453125,
|
||
|
|
"logps/rejected": -392.3923828125,
|
||
|
|
"loss": 0.27,
|
||
|
|
"loss/base_kto": 2.159776210784912,
|
||
|
|
"metrics/advantage_var": 14934.84375,
|
||
|
|
"rewards/chosen": 2.360272026062012,
|
||
|
|
"rewards/margins": 4.595606422424316,
|
||
|
|
"rewards/rejected": -2.2353343963623047,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 7.161087512969971,
|
||
|
|
"kl": 20.279638290405273,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -46109183.23922734,
|
||
|
|
"logits/rejected": -47211197.78583196,
|
||
|
|
"logps/chosen": -464.3273588410104,
|
||
|
|
"logps/rejected": -389.81806013179573,
|
||
|
|
"loss": 0.2719,
|
||
|
|
"loss/base_kto": 2.175161123275757,
|
||
|
|
"metrics/advantage_var": 14320.359375,
|
||
|
|
"rewards/chosen": 2.531824256477526,
|
||
|
|
"rewards/margins": 4.456279826905862,
|
||
|
|
"rewards/rejected": -1.9244555704283361,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 4.648787021636963,
|
||
|
|
"kl": 22.47624397277832,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -48343649.215189874,
|
||
|
|
"logits/rejected": -47700252.44444445,
|
||
|
|
"logps/chosen": -455.1236155063291,
|
||
|
|
"logps/rejected": -385.0729166666667,
|
||
|
|
"loss": 0.255,
|
||
|
|
"loss/base_kto": 2.0397450923919678,
|
||
|
|
"metrics/advantage_var": 13408.5517578125,
|
||
|
|
"rewards/chosen": 2.3995195219788372,
|
||
|
|
"rewards/margins": 4.7239884407674175,
|
||
|
|
"rewards/rejected": -2.3244689187885803,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 8.42148494720459,
|
||
|
|
"kl": 18.592853546142578,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -47214087.78115501,
|
||
|
|
"logits/rejected": -49079750.37942122,
|
||
|
|
"logps/chosen": -466.7443009118541,
|
||
|
|
"logps/rejected": -395.5042453778135,
|
||
|
|
"loss": 0.2471,
|
||
|
|
"loss/base_kto": 1.9770435094833374,
|
||
|
|
"metrics/advantage_var": 15593.9033203125,
|
||
|
|
"rewards/chosen": 2.7582436419547873,
|
||
|
|
"rewards/margins": 5.262467431770904,
|
||
|
|
"rewards/rejected": -2.5042237898161175,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 7.439265727996826,
|
||
|
|
"kl": 11.96108341217041,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -46719675.470769234,
|
||
|
|
"logits/rejected": -48405676.29206349,
|
||
|
|
"logps/chosen": -473.23932692307693,
|
||
|
|
"logps/rejected": -387.12862103174604,
|
||
|
|
"loss": 0.2452,
|
||
|
|
"loss/base_kto": 1.9619816541671753,
|
||
|
|
"metrics/advantage_var": 16483.15625,
|
||
|
|
"rewards/chosen": 2.296476487379808,
|
||
|
|
"rewards/margins": 5.290778710222069,
|
||
|
|
"rewards/rejected": -2.9943022228422618,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 7.7969584465026855,
|
||
|
|
"kl": 32.29633331298828,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -47421925.218461536,
|
||
|
|
"logits/rejected": -48460608.203174606,
|
||
|
|
"logps/chosen": -468.878125,
|
||
|
|
"logps/rejected": -376.4671626984127,
|
||
|
|
"loss": 0.2681,
|
||
|
|
"loss/base_kto": 2.1451966762542725,
|
||
|
|
"metrics/advantage_var": 14988.861328125,
|
||
|
|
"rewards/chosen": 2.5041038161057694,
|
||
|
|
"rewards/margins": 4.649634298806662,
|
||
|
|
"rewards/rejected": -2.145530482700893,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 7.675873279571533,
|
||
|
|
"kl": 29.6217098236084,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -47351358.43902439,
|
||
|
|
"logits/rejected": -49409986.40601504,
|
||
|
|
"logps/chosen": -454.938262195122,
|
||
|
|
"logps/rejected": -417.4201127819549,
|
||
|
|
"loss": 0.2552,
|
||
|
|
"loss/base_kto": 2.0416419506073,
|
||
|
|
"metrics/advantage_var": 15909.7724609375,
|
||
|
|
"rewards/chosen": 2.3638814786585365,
|
||
|
|
"rewards/margins": 5.011706607887859,
|
||
|
|
"rewards/rejected": -2.6478251292293233,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 7.191428184509277,
|
||
|
|
"kl": 25.549884796142578,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -48202389.08485857,
|
||
|
|
"logits/rejected": -47982344.671575844,
|
||
|
|
"logps/chosen": -471.9688019966722,
|
||
|
|
"logps/rejected": -407.2490795287187,
|
||
|
|
"loss": 0.2533,
|
||
|
|
"loss/base_kto": 2.026344060897827,
|
||
|
|
"metrics/advantage_var": 15896.3876953125,
|
||
|
|
"rewards/chosen": 2.4040238924708817,
|
||
|
|
"rewards/margins": 4.864776579096434,
|
||
|
|
"rewards/rejected": -2.4607526866255522,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 8.419614791870117,
|
||
|
|
"kl": 16.007797241210938,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -49458072.95597484,
|
||
|
|
"logits/rejected": -50207880.745341614,
|
||
|
|
"logps/chosen": -478.63674331761007,
|
||
|
|
"logps/rejected": -393.54697204968943,
|
||
|
|
"loss": 0.2544,
|
||
|
|
"loss/base_kto": 2.0351531505584717,
|
||
|
|
"metrics/advantage_var": 15565.5380859375,
|
||
|
|
"rewards/chosen": 2.083188230886399,
|
||
|
|
"rewards/margins": 4.963903168653285,
|
||
|
|
"rewards/rejected": -2.8807149377668866,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 11.954970359802246,
|
||
|
|
"kl": 22.1839542388916,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -48730670.54545455,
|
||
|
|
"logits/rejected": -50354456.67469879,
|
||
|
|
"logps/chosen": -483.3146814123377,
|
||
|
|
"logps/rejected": -423.3035579819277,
|
||
|
|
"loss": 0.2647,
|
||
|
|
"loss/base_kto": 2.1177659034729004,
|
||
|
|
"metrics/advantage_var": 13992.3408203125,
|
||
|
|
"rewards/chosen": 2.2816132384461243,
|
||
|
|
"rewards/margins": 4.731054068171275,
|
||
|
|
"rewards/rejected": -2.4494408297251504,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 8.50296401977539,
|
||
|
|
"kl": 22.303434371948242,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -48743870.03680982,
|
||
|
|
"logits/rejected": -48878109.35031847,
|
||
|
|
"logps/chosen": -458.819018404908,
|
||
|
|
"logps/rejected": -404.78393710191085,
|
||
|
|
"loss": 0.2466,
|
||
|
|
"loss/base_kto": 1.9726238250732422,
|
||
|
|
"metrics/advantage_var": 14543.2841796875,
|
||
|
|
"rewards/chosen": 2.665093357577646,
|
||
|
|
"rewards/margins": 5.108054060267236,
|
||
|
|
"rewards/rejected": -2.44296070268959,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 6.078616619110107,
|
||
|
|
"kl": 23.710634231567383,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -48693167.32121212,
|
||
|
|
"logits/rejected": -50649269.67741936,
|
||
|
|
"logps/chosen": -471.01226325757574,
|
||
|
|
"logps/rejected": -380.1893397177419,
|
||
|
|
"loss": 0.2634,
|
||
|
|
"loss/base_kto": 2.107532501220703,
|
||
|
|
"metrics/advantage_var": 13206.234375,
|
||
|
|
"rewards/chosen": 2.6341057054924244,
|
||
|
|
"rewards/margins": 4.685351807117929,
|
||
|
|
"rewards/rejected": -2.051246101625504,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 8.119199752807617,
|
||
|
|
"kl": 23.26380729675293,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -49470722.77829457,
|
||
|
|
"logits/rejected": -50942855.055118114,
|
||
|
|
"logps/chosen": -468.51017441860466,
|
||
|
|
"logps/rejected": -386.84995078740155,
|
||
|
|
"loss": 0.2521,
|
||
|
|
"loss/base_kto": 2.016773223876953,
|
||
|
|
"metrics/advantage_var": 13627.8154296875,
|
||
|
|
"rewards/chosen": 2.7830914637839146,
|
||
|
|
"rewards/margins": 4.81463481700734,
|
||
|
|
"rewards/rejected": -2.0315433532234253,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 8.171038627624512,
|
||
|
|
"kl": 23.872556686401367,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -50876945.766561516,
|
||
|
|
"logits/rejected": -51860790.6873065,
|
||
|
|
"logps/chosen": -471.33147673501577,
|
||
|
|
"logps/rejected": -376.60531153250776,
|
||
|
|
"loss": 0.2431,
|
||
|
|
"loss/base_kto": 1.9447319507598877,
|
||
|
|
"metrics/advantage_var": 13762.4755859375,
|
||
|
|
"rewards/chosen": 2.4297029031940065,
|
||
|
|
"rewards/margins": 5.0148431668961155,
|
||
|
|
"rewards/rejected": -2.585140263702109,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 8.106106758117676,
|
||
|
|
"kl": 16.061141967773438,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -49467057.63265306,
|
||
|
|
"logits/rejected": -49775573.00155521,
|
||
|
|
"logps/chosen": -453.4469191522763,
|
||
|
|
"logps/rejected": -412.4076594090202,
|
||
|
|
"loss": 0.262,
|
||
|
|
"loss/base_kto": 2.0961430072784424,
|
||
|
|
"metrics/advantage_var": 13220.462890625,
|
||
|
|
"rewards/chosen": 2.22434262166405,
|
||
|
|
"rewards/margins": 4.581668814801686,
|
||
|
|
"rewards/rejected": -2.357326193137636,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 8.971443176269531,
|
||
|
|
"kl": 20.7944393157959,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -51214772.86535552,
|
||
|
|
"logits/rejected": -50312550.978998385,
|
||
|
|
"logps/chosen": -445.6907148260212,
|
||
|
|
"logps/rejected": -378.6051342891761,
|
||
|
|
"loss": 0.2542,
|
||
|
|
"loss/base_kto": 2.0339062213897705,
|
||
|
|
"metrics/advantage_var": 12345.4638671875,
|
||
|
|
"rewards/chosen": 2.8841517646085477,
|
||
|
|
"rewards/margins": 4.582793096393685,
|
||
|
|
"rewards/rejected": -1.6986413317851372,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 8.93839168548584,
|
||
|
|
"kl": 27.64991569519043,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -50742585.2044374,
|
||
|
|
"logits/rejected": -51969685.103235744,
|
||
|
|
"logps/chosen": -455.51550118858955,
|
||
|
|
"logps/rejected": -418.8906009244992,
|
||
|
|
"loss": 0.2555,
|
||
|
|
"loss/base_kto": 2.043959379196167,
|
||
|
|
"metrics/advantage_var": 14285.34375,
|
||
|
|
"rewards/chosen": 2.7165968421899764,
|
||
|
|
"rewards/margins": 4.896977265196525,
|
||
|
|
"rewards/rejected": -2.1803804230065484,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 7.332820892333984,
|
||
|
|
"kl": 27.189178466796875,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -51423125.40063091,
|
||
|
|
"logits/rejected": -50545923.9628483,
|
||
|
|
"logps/chosen": -452.23718454258676,
|
||
|
|
"logps/rejected": -400.2590944272446,
|
||
|
|
"loss": 0.2566,
|
||
|
|
"loss/base_kto": 2.0524163246154785,
|
||
|
|
"metrics/advantage_var": 14592.6845703125,
|
||
|
|
"rewards/chosen": 2.5982067216458007,
|
||
|
|
"rewards/margins": 4.938431799649864,
|
||
|
|
"rewards/rejected": -2.3402250780040634,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 7.5371832847595215,
|
||
|
|
"kl": 19.167118072509766,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -49825617.63467492,
|
||
|
|
"logits/rejected": -51343172.643533126,
|
||
|
|
"logps/chosen": -486.61992066563465,
|
||
|
|
"logps/rejected": -386.4467665615142,
|
||
|
|
"loss": 0.2356,
|
||
|
|
"loss/base_kto": 1.8851795196533203,
|
||
|
|
"metrics/advantage_var": 14707.625,
|
||
|
|
"rewards/chosen": 2.4065221071981426,
|
||
|
|
"rewards/margins": 5.221589434559144,
|
||
|
|
"rewards/rejected": -2.8150673273610014,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 9.608060836791992,
|
||
|
|
"kl": 22.010770797729492,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -51650882.49350649,
|
||
|
|
"logits/rejected": -51695082.409638554,
|
||
|
|
"logps/chosen": -469.02404626623377,
|
||
|
|
"logps/rejected": -414.7276920180723,
|
||
|
|
"loss": 0.2506,
|
||
|
|
"loss/base_kto": 2.00508451461792,
|
||
|
|
"metrics/advantage_var": 16096.7578125,
|
||
|
|
"rewards/chosen": 2.6388319932021105,
|
||
|
|
"rewards/margins": 5.131839974835676,
|
||
|
|
"rewards/rejected": -2.4930079816335655,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 7.547207355499268,
|
||
|
|
"kl": 19.215185165405273,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -48986785.77198697,
|
||
|
|
"logits/rejected": -50203435.81981982,
|
||
|
|
"logps/chosen": -476.5796009771987,
|
||
|
|
"logps/rejected": -382.5185341591592,
|
||
|
|
"loss": 0.2648,
|
||
|
|
"loss/base_kto": 2.118427276611328,
|
||
|
|
"metrics/advantage_var": 15124.556640625,
|
||
|
|
"rewards/chosen": 2.5517281895740025,
|
||
|
|
"rewards/margins": 4.824689871959513,
|
||
|
|
"rewards/rejected": -2.2729616823855103,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 8.951262474060059,
|
||
|
|
"kl": 19.080490112304688,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -52171221.40031397,
|
||
|
|
"logits/rejected": -52181638.56920684,
|
||
|
|
"logps/chosen": -454.0738324175824,
|
||
|
|
"logps/rejected": -409.19012441679627,
|
||
|
|
"loss": 0.2503,
|
||
|
|
"loss/base_kto": 2.002168655395508,
|
||
|
|
"metrics/advantage_var": 16247.0625,
|
||
|
|
"rewards/chosen": 2.1811902871124413,
|
||
|
|
"rewards/margins": 5.068549574024377,
|
||
|
|
"rewards/rejected": -2.8873592869119364,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 7.5546159744262695,
|
||
|
|
"kl": 16.825531005859375,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -51035750.4,
|
||
|
|
"logits/rejected": -50864886.065671645,
|
||
|
|
"logps/chosen": -432.0246413934426,
|
||
|
|
"logps/rejected": -381.96408582089555,
|
||
|
|
"loss": 0.249,
|
||
|
|
"loss/base_kto": 1.9919849634170532,
|
||
|
|
"metrics/advantage_var": 15307.09375,
|
||
|
|
"rewards/chosen": 2.4189148949795083,
|
||
|
|
"rewards/margins": 5.1869454089137434,
|
||
|
|
"rewards/rejected": -2.768030513934235,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 9.785651206970215,
|
||
|
|
"kl": 14.184760093688965,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -50667513.88656716,
|
||
|
|
"logits/rejected": -51588018.78032787,
|
||
|
|
"logps/chosen": -462.3922574626866,
|
||
|
|
"logps/rejected": -409.36296106557376,
|
||
|
|
"loss": 0.267,
|
||
|
|
"loss/base_kto": 2.1362154483795166,
|
||
|
|
"metrics/advantage_var": 15636.0283203125,
|
||
|
|
"rewards/chosen": 2.073510013409515,
|
||
|
|
"rewards/margins": 4.814888647502753,
|
||
|
|
"rewards/rejected": -2.7413786340932376,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 7.976250648498535,
|
||
|
|
"kl": 29.88347816467285,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -50107616.859399684,
|
||
|
|
"logits/rejected": -49035365.29211746,
|
||
|
|
"logps/chosen": -424.58046998420224,
|
||
|
|
"logps/rejected": -370.0915282071097,
|
||
|
|
"loss": 0.2641,
|
||
|
|
"loss/base_kto": 2.1128010749816895,
|
||
|
|
"metrics/advantage_var": 13511.4091796875,
|
||
|
|
"rewards/chosen": 2.773539900214751,
|
||
|
|
"rewards/margins": 4.7543954263208175,
|
||
|
|
"rewards/rejected": -1.9808555261060665,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 7.568881988525391,
|
||
|
|
"kl": 19.313095092773438,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -49129610.9022082,
|
||
|
|
"logits/rejected": -48939949.57275542,
|
||
|
|
"logps/chosen": -440.9421332807571,
|
||
|
|
"logps/rejected": -370.9177147832817,
|
||
|
|
"loss": 0.2474,
|
||
|
|
"loss/base_kto": 1.9794552326202393,
|
||
|
|
"metrics/advantage_var": 14084.15625,
|
||
|
|
"rewards/chosen": 2.604433655362776,
|
||
|
|
"rewards/margins": 4.982342934822915,
|
||
|
|
"rewards/rejected": -2.3779092794601393,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 6.2476887702941895,
|
||
|
|
"kl": 13.618782043457031,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -50499784.8488746,
|
||
|
|
"logits/rejected": -51509512.559270516,
|
||
|
|
"logps/chosen": -465.0994774919614,
|
||
|
|
"logps/rejected": -411.11241451367783,
|
||
|
|
"loss": 0.2641,
|
||
|
|
"loss/base_kto": 2.1127593517303467,
|
||
|
|
"metrics/advantage_var": 16641.234375,
|
||
|
|
"rewards/chosen": 2.1833678610455185,
|
||
|
|
"rewards/margins": 4.954416641678117,
|
||
|
|
"rewards/rejected": -2.771048780632599,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 8.074265480041504,
|
||
|
|
"kl": 20.739089965820312,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -51888706.983458646,
|
||
|
|
"logits/rejected": -51872393.36585366,
|
||
|
|
"logps/chosen": -473.73853383458646,
|
||
|
|
"logps/rejected": -398.342987804878,
|
||
|
|
"loss": 0.257,
|
||
|
|
"loss/base_kto": 2.05631422996521,
|
||
|
|
"metrics/advantage_var": 14822.2880859375,
|
||
|
|
"rewards/chosen": 2.704866291705827,
|
||
|
|
"rewards/margins": 4.915345680679405,
|
||
|
|
"rewards/rejected": -2.210479388973577,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 9.068573951721191,
|
||
|
|
"kl": 21.232534408569336,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -52277266.70319635,
|
||
|
|
"logits/rejected": -51626911.02407704,
|
||
|
|
"logps/chosen": -485.71004566210047,
|
||
|
|
"logps/rejected": -399.7011687399679,
|
||
|
|
"loss": 0.2501,
|
||
|
|
"loss/base_kto": 2.0007503032684326,
|
||
|
|
"metrics/advantage_var": 15226.5771484375,
|
||
|
|
"rewards/chosen": 2.7049560546875,
|
||
|
|
"rewards/margins": 5.207591441048857,
|
||
|
|
"rewards/rejected": -2.5026353863613564,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 9.576692581176758,
|
||
|
|
"kl": 28.596731185913086,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -50640729.6,
|
||
|
|
"logits/rejected": -50948531.2,
|
||
|
|
"logps/chosen": -447.75751953125,
|
||
|
|
"logps/rejected": -390.83759765625,
|
||
|
|
"loss": 0.2562,
|
||
|
|
"loss/base_kto": 2.0498600006103516,
|
||
|
|
"metrics/advantage_var": 14499.0576171875,
|
||
|
|
"rewards/chosen": 2.763779067993164,
|
||
|
|
"rewards/margins": 4.837852859497071,
|
||
|
|
"rewards/rejected": -2.0740737915039062,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 8.336225509643555,
|
||
|
|
"kl": 21.048917770385742,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -51244268.057233706,
|
||
|
|
"logits/rejected": -53232616.40552995,
|
||
|
|
"logps/chosen": -446.7321641494436,
|
||
|
|
"logps/rejected": -404.64564132104454,
|
||
|
|
"loss": 0.2442,
|
||
|
|
"loss/base_kto": 1.9533166885375977,
|
||
|
|
"metrics/advantage_var": 15715.361328125,
|
||
|
|
"rewards/chosen": 2.4377893590396464,
|
||
|
|
"rewards/margins": 5.201704249592642,
|
||
|
|
"rewards/rejected": -2.7639148905529956,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 7.756782054901123,
|
||
|
|
"kl": 23.05873680114746,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -53175127.69278034,
|
||
|
|
"logits/rejected": -52764330.12400636,
|
||
|
|
"logps/chosen": -454.0267857142857,
|
||
|
|
"logps/rejected": -410.45588235294116,
|
||
|
|
"loss": 0.2512,
|
||
|
|
"loss/base_kto": 2.009918451309204,
|
||
|
|
"metrics/advantage_var": 14705.5224609375,
|
||
|
|
"rewards/chosen": 2.501078194004416,
|
||
|
|
"rewards/margins": 5.026136570395513,
|
||
|
|
"rewards/rejected": -2.525058376391097,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 10.919984817504883,
|
||
|
|
"kl": 31.348316192626953,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -52322684.1088,
|
||
|
|
"logits/rejected": -51838244.3480916,
|
||
|
|
"logps/chosen": -434.9367,
|
||
|
|
"logps/rejected": -396.92793416030537,
|
||
|
|
"loss": 0.2649,
|
||
|
|
"loss/base_kto": 2.1192855834960938,
|
||
|
|
"metrics/advantage_var": 15045.2470703125,
|
||
|
|
"rewards/chosen": 2.8001419921875,
|
||
|
|
"rewards/margins": 4.800822044966603,
|
||
|
|
"rewards/rejected": -2.000680052779103,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 9.527816772460938,
|
||
|
|
"kl": 12.20826530456543,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -51634665.3164557,
|
||
|
|
"logits/rejected": -52495334.71604938,
|
||
|
|
"logps/chosen": -464.9095628955696,
|
||
|
|
"logps/rejected": -403.5522762345679,
|
||
|
|
"loss": 0.2516,
|
||
|
|
"loss/base_kto": 2.012443780899048,
|
||
|
|
"metrics/advantage_var": 15119.6943359375,
|
||
|
|
"rewards/chosen": 2.351209616359276,
|
||
|
|
"rewards/margins": 5.110908178039446,
|
||
|
|
"rewards/rejected": -2.7596985616801697,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 9.87900447845459,
|
||
|
|
"kl": 17.386611938476562,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -53422185.43525741,
|
||
|
|
"logits/rejected": -53120264.41314554,
|
||
|
|
"logps/chosen": -482.53310257410294,
|
||
|
|
"logps/rejected": -407.90179968701096,
|
||
|
|
"loss": 0.2483,
|
||
|
|
"loss/base_kto": 1.986655592918396,
|
||
|
|
"metrics/advantage_var": 16186.7998046875,
|
||
|
|
"rewards/chosen": 2.3212818258824104,
|
||
|
|
"rewards/margins": 5.302370479112556,
|
||
|
|
"rewards/rejected": -2.981088653230145,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 6.876319408416748,
|
||
|
|
"kl": 27.84088706970215,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -52926714.064914994,
|
||
|
|
"logits/rejected": -54163553.06161138,
|
||
|
|
"logps/chosen": -463.3681897217929,
|
||
|
|
"logps/rejected": -392.51132997630333,
|
||
|
|
"loss": 0.2423,
|
||
|
|
"loss/base_kto": 1.9387661218643188,
|
||
|
|
"metrics/advantage_var": 15857.3505859375,
|
||
|
|
"rewards/chosen": 2.6382854762364762,
|
||
|
|
"rewards/margins": 5.196615029825536,
|
||
|
|
"rewards/rejected": -2.55832955358906,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 9.612847328186035,
|
||
|
|
"kl": 19.885042190551758,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -50576158.91021672,
|
||
|
|
"logits/rejected": -52167256.83280757,
|
||
|
|
"logps/chosen": -458.50880417956654,
|
||
|
|
"logps/rejected": -413.5228213722398,
|
||
|
|
"loss": 0.2445,
|
||
|
|
"loss/base_kto": 1.955782175064087,
|
||
|
|
"metrics/advantage_var": 16214.2626953125,
|
||
|
|
"rewards/chosen": 2.4565017747436144,
|
||
|
|
"rewards/margins": 5.124602222052369,
|
||
|
|
"rewards/rejected": -2.668100447308754,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 7.085169315338135,
|
||
|
|
"kl": 19.323680877685547,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -51409870.65060241,
|
||
|
|
"logits/rejected": -51447382.44155844,
|
||
|
|
"logps/chosen": -442.01571912650604,
|
||
|
|
"logps/rejected": -404.3957234172078,
|
||
|
|
"loss": 0.2585,
|
||
|
|
"loss/base_kto": 2.0679147243499756,
|
||
|
|
"metrics/advantage_var": 14929.962890625,
|
||
|
|
"rewards/chosen": 2.428604861339891,
|
||
|
|
"rewards/margins": 4.838879812828832,
|
||
|
|
"rewards/rejected": -2.4102749514889408,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 5.093369007110596,
|
||
|
|
"kl": 21.76449203491211,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -51313167.176287055,
|
||
|
|
"logits/rejected": -51953249.25588697,
|
||
|
|
"logps/chosen": -461.6126657566303,
|
||
|
|
"logps/rejected": -384.9188088697017,
|
||
|
|
"loss": 0.2277,
|
||
|
|
"loss/base_kto": 1.8218530416488647,
|
||
|
|
"metrics/advantage_var": 15466.078125,
|
||
|
|
"rewards/chosen": 3.0937618071129096,
|
||
|
|
"rewards/margins": 5.48768481351499,
|
||
|
|
"rewards/rejected": -2.3939230064020802,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 6.250664234161377,
|
||
|
|
"kl": 11.092265129089355,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -52543289.6,
|
||
|
|
"logits/rejected": -52555180.8,
|
||
|
|
"logps/chosen": -448.8865234375,
|
||
|
|
"logps/rejected": -426.451171875,
|
||
|
|
"loss": 0.2125,
|
||
|
|
"loss/base_kto": 1.699960708618164,
|
||
|
|
"metrics/advantage_var": 17586.404296875,
|
||
|
|
"rewards/chosen": 2.5158063888549806,
|
||
|
|
"rewards/margins": 5.949070167541504,
|
||
|
|
"rewards/rejected": -3.4332637786865234,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 7.516881942749023,
|
||
|
|
"kl": 27.575057983398438,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -52040782.51757188,
|
||
|
|
"logits/rejected": -53780530.103975534,
|
||
|
|
"logps/chosen": -457.1747703674121,
|
||
|
|
"logps/rejected": -412.95508409785936,
|
||
|
|
"loss": 0.2147,
|
||
|
|
"loss/base_kto": 1.7175458669662476,
|
||
|
|
"metrics/advantage_var": 15806.40625,
|
||
|
|
"rewards/chosen": 2.945287149935104,
|
||
|
|
"rewards/margins": 5.667440186536786,
|
||
|
|
"rewards/rejected": -2.722153036601682,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 7.4503560066223145,
|
||
|
|
"kl": 22.494722366333008,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -53130994.61172742,
|
||
|
|
"logits/rejected": -53505965.95377504,
|
||
|
|
"logps/chosen": -474.13960974643425,
|
||
|
|
"logps/rejected": -411.5372688751926,
|
||
|
|
"loss": 0.2132,
|
||
|
|
"loss/base_kto": 1.7054146528244019,
|
||
|
|
"metrics/advantage_var": 17508.853515625,
|
||
|
|
"rewards/chosen": 2.9541907454065965,
|
||
|
|
"rewards/margins": 5.814634848112682,
|
||
|
|
"rewards/rejected": -2.860444102706086,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 10.237329483032227,
|
||
|
|
"kl": 13.12766170501709,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -53387424.87009063,
|
||
|
|
"logits/rejected": -52597710.291262135,
|
||
|
|
"logps/chosen": -483.8654645015106,
|
||
|
|
"logps/rejected": -389.9562601132686,
|
||
|
|
"loss": 0.2036,
|
||
|
|
"loss/base_kto": 1.6287109851837158,
|
||
|
|
"metrics/advantage_var": 14979.9921875,
|
||
|
|
"rewards/chosen": 2.758353149782855,
|
||
|
|
"rewards/margins": 5.7842583268282635,
|
||
|
|
"rewards/rejected": -3.0259051770454084,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 7.691054344177246,
|
||
|
|
"kl": 18.150373458862305,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -51640392.803709425,
|
||
|
|
"logits/rejected": -53083939.99368089,
|
||
|
|
"logps/chosen": -438.0470923493045,
|
||
|
|
"logps/rejected": -424.92683649289097,
|
||
|
|
"loss": 0.2083,
|
||
|
|
"loss/base_kto": 1.6663763523101807,
|
||
|
|
"metrics/advantage_var": 17248.298828125,
|
||
|
|
"rewards/chosen": 2.568194098966383,
|
||
|
|
"rewards/margins": 5.949468088771873,
|
||
|
|
"rewards/rejected": -3.3812739898054898,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 6.853572845458984,
|
||
|
|
"kl": 15.213935852050781,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -53337403.43550835,
|
||
|
|
"logits/rejected": -54221581.60386474,
|
||
|
|
"logps/chosen": -459.4839719271624,
|
||
|
|
"logps/rejected": -401.6209490740741,
|
||
|
|
"loss": 0.2244,
|
||
|
|
"loss/base_kto": 1.794970154762268,
|
||
|
|
"metrics/advantage_var": 14274.5849609375,
|
||
|
|
"rewards/chosen": 2.6967603020675264,
|
||
|
|
"rewards/margins": 5.361152647452993,
|
||
|
|
"rewards/rejected": -2.664392345385467,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 6.441402912139893,
|
||
|
|
"kl": 16.609342575073242,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -51576548.280254774,
|
||
|
|
"logits/rejected": -52661995.582822084,
|
||
|
|
"logps/chosen": -469.22103901273886,
|
||
|
|
"logps/rejected": -389.69545628834356,
|
||
|
|
"loss": 0.205,
|
||
|
|
"loss/base_kto": 1.6403454542160034,
|
||
|
|
"metrics/advantage_var": 16205.587890625,
|
||
|
|
"rewards/chosen": 2.563711761668989,
|
||
|
|
"rewards/margins": 5.923686628664771,
|
||
|
|
"rewards/rejected": -3.359974866995782,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 8.55733871459961,
|
||
|
|
"kl": 25.37789535522461,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -52256164.345679015,
|
||
|
|
"logits/rejected": -52587818.12658228,
|
||
|
|
"logps/chosen": -493.7014371141975,
|
||
|
|
"logps/rejected": -412.27558840981015,
|
||
|
|
"loss": 0.2122,
|
||
|
|
"loss/base_kto": 1.6972593069076538,
|
||
|
|
"metrics/advantage_var": 16703.544921875,
|
||
|
|
"rewards/chosen": 2.814942913290895,
|
||
|
|
"rewards/margins": 5.731951325326101,
|
||
|
|
"rewards/rejected": -2.9170084120352056,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 8.511039733886719,
|
||
|
|
"kl": 16.398456573486328,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -51757954.96754251,
|
||
|
|
"logits/rejected": -52324258.17377567,
|
||
|
|
"logps/chosen": -449.0143933539413,
|
||
|
|
"logps/rejected": -397.59809439178514,
|
||
|
|
"loss": 0.2144,
|
||
|
|
"loss/base_kto": 1.7153100967407227,
|
||
|
|
"metrics/advantage_var": 15016.7314453125,
|
||
|
|
"rewards/chosen": 2.92261194998551,
|
||
|
|
"rewards/margins": 5.685572423116928,
|
||
|
|
"rewards/rejected": -2.7629604731314177,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 7.376357078552246,
|
||
|
|
"kl": 16.149038314819336,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -52741254.69413629,
|
||
|
|
"logits/rejected": -54535728.12326656,
|
||
|
|
"logps/chosen": -455.31596671949285,
|
||
|
|
"logps/rejected": -397.3636122881356,
|
||
|
|
"loss": 0.2114,
|
||
|
|
"loss/base_kto": 1.6911267042160034,
|
||
|
|
"metrics/advantage_var": 17047.3984375,
|
||
|
|
"rewards/chosen": 2.559685998786648,
|
||
|
|
"rewards/margins": 5.678751011667041,
|
||
|
|
"rewards/rejected": -3.1190650128803927,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 7.0808024406433105,
|
||
|
|
"kl": 19.327743530273438,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -51179646.95595432,
|
||
|
|
"logits/rejected": -52681646.63268366,
|
||
|
|
"logps/chosen": -452.35924755301795,
|
||
|
|
"logps/rejected": -393.6358695652174,
|
||
|
|
"loss": 0.2079,
|
||
|
|
"loss/base_kto": 1.6628249883651733,
|
||
|
|
"metrics/advantage_var": 17884.87890625,
|
||
|
|
"rewards/chosen": 2.5431923796263254,
|
||
|
|
"rewards/margins": 5.888858169778125,
|
||
|
|
"rewards/rejected": -3.345665790151799,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 7.206068515777588,
|
||
|
|
"kl": 13.962677001953125,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -52635359.09717868,
|
||
|
|
"logits/rejected": -52769335.82554517,
|
||
|
|
"logps/chosen": -459.94303487460814,
|
||
|
|
"logps/rejected": -409.59657320872276,
|
||
|
|
"loss": 0.2112,
|
||
|
|
"loss/base_kto": 1.6896650791168213,
|
||
|
|
"metrics/advantage_var": 16756.390625,
|
||
|
|
"rewards/chosen": 2.7216389336182405,
|
||
|
|
"rewards/margins": 5.849456438120772,
|
||
|
|
"rewards/rejected": -3.127817504502531,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 7.377704620361328,
|
||
|
|
"kl": 22.29719352722168,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -54559073.429892145,
|
||
|
|
"logits/rejected": -54657004.52614897,
|
||
|
|
"logps/chosen": -472.5718412942989,
|
||
|
|
"logps/rejected": -420.8526148969889,
|
||
|
|
"loss": 0.2174,
|
||
|
|
"loss/base_kto": 1.7388498783111572,
|
||
|
|
"metrics/advantage_var": 15162.697265625,
|
||
|
|
"rewards/chosen": 2.6641059487552967,
|
||
|
|
"rewards/margins": 5.5746843182828325,
|
||
|
|
"rewards/rejected": -2.910578369527536,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 6.108307361602783,
|
||
|
|
"kl": 18.389760971069336,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -51068998.73155416,
|
||
|
|
"logits/rejected": -52572131.334370136,
|
||
|
|
"logps/chosen": -464.33035714285717,
|
||
|
|
"logps/rejected": -417.0138024883359,
|
||
|
|
"loss": 0.2056,
|
||
|
|
"loss/base_kto": 1.645011305809021,
|
||
|
|
"metrics/advantage_var": 15969.048828125,
|
||
|
|
"rewards/chosen": 2.8435683317798275,
|
||
|
|
"rewards/margins": 5.9239052154671334,
|
||
|
|
"rewards/rejected": -3.0803368836873055,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 7.382128715515137,
|
||
|
|
"kl": 11.044879913330078,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -51034927.22330097,
|
||
|
|
"logits/rejected": -51440080.04833837,
|
||
|
|
"logps/chosen": -439.081715210356,
|
||
|
|
"logps/rejected": -402.0004248489426,
|
||
|
|
"loss": 0.2036,
|
||
|
|
"loss/base_kto": 1.6291121244430542,
|
||
|
|
"metrics/advantage_var": 15075.765625,
|
||
|
|
"rewards/chosen": 2.762153502035295,
|
||
|
|
"rewards/margins": 5.691052892495076,
|
||
|
|
"rewards/rejected": -2.928899390459781,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 7.5083699226379395,
|
||
|
|
"kl": 9.02786922454834,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -51969928.23391813,
|
||
|
|
"logits/rejected": -53152341.906040266,
|
||
|
|
"logps/chosen": -435.16282894736844,
|
||
|
|
"logps/rejected": -393.66335465604027,
|
||
|
|
"loss": 0.2005,
|
||
|
|
"loss/base_kto": 1.6041138172149658,
|
||
|
|
"metrics/advantage_var": 16324.6689453125,
|
||
|
|
"rewards/chosen": 2.6327168425621346,
|
||
|
|
"rewards/margins": 5.971829809214819,
|
||
|
|
"rewards/rejected": -3.3391129666526846,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 6.553695201873779,
|
||
|
|
"kl": 22.991432189941406,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -52758514.76575121,
|
||
|
|
"logits/rejected": -52305994.36006051,
|
||
|
|
"logps/chosen": -456.5091377221325,
|
||
|
|
"logps/rejected": -399.68253593040845,
|
||
|
|
"loss": 0.2148,
|
||
|
|
"loss/base_kto": 1.7184559106826782,
|
||
|
|
"metrics/advantage_var": 15160.0341796875,
|
||
|
|
"rewards/chosen": 2.5292734075247374,
|
||
|
|
"rewards/margins": 5.505444026733323,
|
||
|
|
"rewards/rejected": -2.9761706192085855,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 7.786435127258301,
|
||
|
|
"kl": 11.857638359069824,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -52542490.496194825,
|
||
|
|
"logits/rejected": -53520084.85393258,
|
||
|
|
"logps/chosen": -461.29119101978694,
|
||
|
|
"logps/rejected": -406.9080557784912,
|
||
|
|
"loss": 0.2095,
|
||
|
|
"loss/base_kto": 1.675807237625122,
|
||
|
|
"metrics/advantage_var": 16317.4560546875,
|
||
|
|
"rewards/chosen": 2.6273080021641935,
|
||
|
|
"rewards/margins": 5.775256437156168,
|
||
|
|
"rewards/rejected": -3.1479484349919744,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 8.449359893798828,
|
||
|
|
"kl": 22.929365158081055,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -53763288.906403944,
|
||
|
|
"logits/rejected": -54059221.65126677,
|
||
|
|
"logps/chosen": -457.6953509852217,
|
||
|
|
"logps/rejected": -407.5998975409836,
|
||
|
|
"loss": 0.2096,
|
||
|
|
"loss/base_kto": 1.6766414642333984,
|
||
|
|
"metrics/advantage_var": 15792.712890625,
|
||
|
|
"rewards/chosen": 2.634127812628284,
|
||
|
|
"rewards/margins": 5.735816203323143,
|
||
|
|
"rewards/rejected": -3.1016883906948585,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 8.70055866241455,
|
||
|
|
"kl": 14.306711196899414,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -52797474.7654321,
|
||
|
|
"logits/rejected": -53016200.101265825,
|
||
|
|
"logps/chosen": -480.6525848765432,
|
||
|
|
"logps/rejected": -392.47201344936707,
|
||
|
|
"loss": 0.1963,
|
||
|
|
"loss/base_kto": 1.5704221725463867,
|
||
|
|
"metrics/advantage_var": 15938.1923828125,
|
||
|
|
"rewards/chosen": 2.8278782220534335,
|
||
|
|
"rewards/margins": 5.77576037939125,
|
||
|
|
"rewards/rejected": -2.9478821573378164,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 8.104622840881348,
|
||
|
|
"kl": 15.392233848571777,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -51529518.4372093,
|
||
|
|
"logits/rejected": -53984957.48031496,
|
||
|
|
"logps/chosen": -461.63788759689925,
|
||
|
|
"logps/rejected": -396.53041338582676,
|
||
|
|
"loss": 0.1926,
|
||
|
|
"loss/base_kto": 1.5410044193267822,
|
||
|
|
"metrics/advantage_var": 16571.763671875,
|
||
|
|
"rewards/chosen": 2.636381116763566,
|
||
|
|
"rewards/margins": 6.110318432216322,
|
||
|
|
"rewards/rejected": -3.4739373154527557,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 6.42228364944458,
|
||
|
|
"kl": 10.110087394714355,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -51332418.37037037,
|
||
|
|
"logits/rejected": -52560494.17721519,
|
||
|
|
"logps/chosen": -452.5120563271605,
|
||
|
|
"logps/rejected": -386.1056170886076,
|
||
|
|
"loss": 0.211,
|
||
|
|
"loss/base_kto": 1.6881433725357056,
|
||
|
|
"metrics/advantage_var": 16214.1220703125,
|
||
|
|
"rewards/chosen": 2.722373679832176,
|
||
|
|
"rewards/margins": 5.686483462763347,
|
||
|
|
"rewards/rejected": -2.9641097829311707,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 6.703454971313477,
|
||
|
|
"kl": 16.846250534057617,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -52858190.76923077,
|
||
|
|
"logits/rejected": -53259863.414634146,
|
||
|
|
"logps/chosen": -457.1239983974359,
|
||
|
|
"logps/rejected": -412.1909298780488,
|
||
|
|
"loss": 0.2052,
|
||
|
|
"loss/base_kto": 1.641872763633728,
|
||
|
|
"metrics/advantage_var": 17799.373046875,
|
||
|
|
"rewards/chosen": 2.7151457957732372,
|
||
|
|
"rewards/margins": 6.064503959002087,
|
||
|
|
"rewards/rejected": -3.349358163228849,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 6.7395477294921875,
|
||
|
|
"kl": 16.90166664123535,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -52330051.62264151,
|
||
|
|
"logits/rejected": -53894061.31677019,
|
||
|
|
"logps/chosen": -482.8420794025157,
|
||
|
|
"logps/rejected": -388.3456424689441,
|
||
|
|
"loss": 0.2216,
|
||
|
|
"loss/base_kto": 1.7730960845947266,
|
||
|
|
"metrics/advantage_var": 15315.8974609375,
|
||
|
|
"rewards/chosen": 2.178255093172661,
|
||
|
|
"rewards/margins": 5.402397416542227,
|
||
|
|
"rewards/rejected": -3.2241423233695654,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 6.1936163902282715,
|
||
|
|
"kl": 11.923944473266602,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -51702307.17557252,
|
||
|
|
"logits/rejected": -51026467.2256,
|
||
|
|
"logps/chosen": -469.8072041984733,
|
||
|
|
"logps/rejected": -390.839975,
|
||
|
|
"loss": 0.2053,
|
||
|
|
"loss/base_kto": 1.6426628828048706,
|
||
|
|
"metrics/advantage_var": 15595.1533203125,
|
||
|
|
"rewards/chosen": 2.7488445252862594,
|
||
|
|
"rewards/margins": 5.67882479872376,
|
||
|
|
"rewards/rejected": -2.9299802734375,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 8.137475967407227,
|
||
|
|
"kl": 18.132556915283203,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -52089820.04274809,
|
||
|
|
"logits/rejected": -52295008.256,
|
||
|
|
"logps/chosen": -460.9177480916031,
|
||
|
|
"logps/rejected": -393.893775,
|
||
|
|
"loss": 0.2014,
|
||
|
|
"loss/base_kto": 1.6112679243087769,
|
||
|
|
"metrics/advantage_var": 17538.55078125,
|
||
|
|
"rewards/chosen": 2.5705998777433208,
|
||
|
|
"rewards/margins": 6.143112377743321,
|
||
|
|
"rewards/rejected": -3.5725125,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 8.598886489868164,
|
||
|
|
"kl": 16.467830657958984,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -52820833.26443769,
|
||
|
|
"logits/rejected": -53313503.073954985,
|
||
|
|
"logps/chosen": -429.13169642857144,
|
||
|
|
"logps/rejected": -384.4918609324759,
|
||
|
|
"loss": 0.2201,
|
||
|
|
"loss/base_kto": 1.7607765197753906,
|
||
|
|
"metrics/advantage_var": 14958.587890625,
|
||
|
|
"rewards/chosen": 2.4377450682109614,
|
||
|
|
"rewards/margins": 5.479708406296271,
|
||
|
|
"rewards/rejected": -3.0419633380853095,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 7.395063400268555,
|
||
|
|
"kl": 12.96544361114502,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -53403948.905547224,
|
||
|
|
"logits/rejected": -55372835.079934746,
|
||
|
|
"logps/chosen": -457.15821776611693,
|
||
|
|
"logps/rejected": -412.7927966965742,
|
||
|
|
"loss": 0.2175,
|
||
|
|
"loss/base_kto": 1.7399282455444336,
|
||
|
|
"metrics/advantage_var": 15877.1318359375,
|
||
|
|
"rewards/chosen": 2.6581939743019114,
|
||
|
|
"rewards/margins": 5.8172340675293785,
|
||
|
|
"rewards/rejected": -3.1590400932274676,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 5.516819477081299,
|
||
|
|
"kl": 13.18786334991455,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -53165266.95939849,
|
||
|
|
"logits/rejected": -52912474.328455284,
|
||
|
|
"logps/chosen": -427.49783834586464,
|
||
|
|
"logps/rejected": -416.99463922764227,
|
||
|
|
"loss": 0.2309,
|
||
|
|
"loss/base_kto": 1.847333550453186,
|
||
|
|
"metrics/advantage_var": 15203.5458984375,
|
||
|
|
"rewards/chosen": 2.697485902255639,
|
||
|
|
"rewards/margins": 5.31505747714131,
|
||
|
|
"rewards/rejected": -2.617571574885671,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 8.08092975616455,
|
||
|
|
"kl": 24.771360397338867,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -52021904.905660376,
|
||
|
|
"logits/rejected": -52015479.254658386,
|
||
|
|
"logps/chosen": -466.30984669811323,
|
||
|
|
"logps/rejected": -414.8922748447205,
|
||
|
|
"loss": 0.2214,
|
||
|
|
"loss/base_kto": 1.7714513540267944,
|
||
|
|
"metrics/advantage_var": 15380.9560546875,
|
||
|
|
"rewards/chosen": 2.7778289602987423,
|
||
|
|
"rewards/margins": 5.55659945446363,
|
||
|
|
"rewards/rejected": -2.7787704941648874,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 6.716619968414307,
|
||
|
|
"kl": 17.462905883789062,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -50473313.774960384,
|
||
|
|
"logits/rejected": -50735018.798151,
|
||
|
|
"logps/chosen": -463.6503565768621,
|
||
|
|
"logps/rejected": -398.61529757318954,
|
||
|
|
"loss": 0.1975,
|
||
|
|
"loss/base_kto": 1.579751968383789,
|
||
|
|
"metrics/advantage_var": 15571.666015625,
|
||
|
|
"rewards/chosen": 2.759819985947405,
|
||
|
|
"rewards/margins": 5.925720123779646,
|
||
|
|
"rewards/rejected": -3.165900137832242,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 7.008581161499023,
|
||
|
|
"kl": 23.658918380737305,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -51330135.21766561,
|
||
|
|
"logits/rejected": -53182067.71517028,
|
||
|
|
"logps/chosen": -464.6318020504732,
|
||
|
|
"logps/rejected": -398.1510739164087,
|
||
|
|
"loss": 0.213,
|
||
|
|
"loss/base_kto": 1.7040843963623047,
|
||
|
|
"metrics/advantage_var": 13738.359375,
|
||
|
|
"rewards/chosen": 2.749936076744874,
|
||
|
|
"rewards/margins": 5.473095347134773,
|
||
|
|
"rewards/rejected": -2.7231592703898992,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 6.6605424880981445,
|
||
|
|
"kl": 15.576194763183594,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -51471504.935384616,
|
||
|
|
"logits/rejected": -52842918.603174604,
|
||
|
|
"logps/chosen": -452.46923076923076,
|
||
|
|
"logps/rejected": -405.70731646825396,
|
||
|
|
"loss": 0.2131,
|
||
|
|
"loss/base_kto": 1.7047852277755737,
|
||
|
|
"metrics/advantage_var": 15001.361328125,
|
||
|
|
"rewards/chosen": 2.5818241060697114,
|
||
|
|
"rewards/margins": 5.591657935434791,
|
||
|
|
"rewards/rejected": -3.0098338293650793,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 10.156596183776855,
|
||
|
|
"kl": 21.255380630493164,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -50288277.02113821,
|
||
|
|
"logits/rejected": -52447093.41353384,
|
||
|
|
"logps/chosen": -464.1432926829268,
|
||
|
|
"logps/rejected": -394.63082706766914,
|
||
|
|
"loss": 0.1964,
|
||
|
|
"loss/base_kto": 1.5715006589889526,
|
||
|
|
"metrics/advantage_var": 15936.7177734375,
|
||
|
|
"rewards/chosen": 2.7470715034298783,
|
||
|
|
"rewards/margins": 5.898373709139465,
|
||
|
|
"rewards/rejected": -3.1513022057095865,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 7.635325908660889,
|
||
|
|
"kl": 15.98388957977295,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -52831451.777424484,
|
||
|
|
"logits/rejected": -54620543.80337942,
|
||
|
|
"logps/chosen": -443.12117448330685,
|
||
|
|
"logps/rejected": -405.9448924731183,
|
||
|
|
"loss": 0.2129,
|
||
|
|
"loss/base_kto": 1.7034294605255127,
|
||
|
|
"metrics/advantage_var": 15774.0283203125,
|
||
|
|
"rewards/chosen": 2.692273356766693,
|
||
|
|
"rewards/margins": 5.745154178639293,
|
||
|
|
"rewards/rejected": -3.0528808218726,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 9.039216995239258,
|
||
|
|
"kl": 18.50642204284668,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -54375990.36450079,
|
||
|
|
"logits/rejected": -54110608.7642527,
|
||
|
|
"logps/chosen": -487.3832210776545,
|
||
|
|
"logps/rejected": -377.65841679506934,
|
||
|
|
"loss": 0.2132,
|
||
|
|
"loss/base_kto": 1.7055686712265015,
|
||
|
|
"metrics/advantage_var": 14751.453125,
|
||
|
|
"rewards/chosen": 2.72312112315521,
|
||
|
|
"rewards/margins": 5.618464733216362,
|
||
|
|
"rewards/rejected": -2.895343610061152,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 8.901484489440918,
|
||
|
|
"kl": 22.53584098815918,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -54056707.63981043,
|
||
|
|
"logits/rejected": -53963812.401854716,
|
||
|
|
"logps/chosen": -465.6426737756714,
|
||
|
|
"logps/rejected": -418.4019513137558,
|
||
|
|
"loss": 0.1967,
|
||
|
|
"loss/base_kto": 1.573914885520935,
|
||
|
|
"metrics/advantage_var": 15834.7001953125,
|
||
|
|
"rewards/chosen": 2.8055148011700237,
|
||
|
|
"rewards/margins": 5.915360535991797,
|
||
|
|
"rewards/rejected": -3.1098457348217736,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.978126922009805e+17,
|
||
|
|
"train_loss": 0.2728243423867102,
|
||
|
|
"train_runtime": 11019.9549,
|
||
|
|
"train_samples_per_second": 13.45,
|
||
|
|
"train_steps_per_second": 0.21
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.978126922009805e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|