46dbd4cae5
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed1 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 12.56043815612793,
|
|
"kl": 11.764090538024902,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36900768.744186044,
|
|
"logits/rejected": -37248082.242519684,
|
|
"logps/chosen": -498.79932170542634,
|
|
"logps/rejected": -360.7293553149606,
|
|
"loss": 0.5315,
|
|
"loss/base_kto": 3.9338440895080566,
|
|
"loss/total_with_kl": 4.252355575561523,
|
|
"metrics/advantage_var": 191.29795837402344,
|
|
"regularization/advantage_var": 191.29795837402344,
|
|
"regularization/kl": 0.31851109862327576,
|
|
"rewards/chosen": 0.1798099251680596,
|
|
"rewards/margins": 0.05079575828352467,
|
|
"rewards/rejected": 0.12901416688453493,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 17.16169548034668,
|
|
"kl": 10.632026672363281,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35866617.73700306,
|
|
"logits/rejected": -37734197.162939295,
|
|
"logps/chosen": -485.1208906727829,
|
|
"logps/rejected": -375.26163138977637,
|
|
"loss": 0.5304,
|
|
"loss/base_kto": 3.914513111114502,
|
|
"loss/total_with_kl": 4.243302345275879,
|
|
"metrics/advantage_var": 197.47079467773438,
|
|
"regularization/advantage_var": 197.47079467773438,
|
|
"regularization/kl": 0.3287888765335083,
|
|
"rewards/chosen": 0.1405060094430906,
|
|
"rewards/margins": 0.06401507097003811,
|
|
"rewards/rejected": 0.0764909384730525,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 28.62113380432129,
|
|
"kl": 26.664426803588867,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -37339925.13761468,
|
|
"logits/rejected": -38068243.629392974,
|
|
"logps/chosen": -465.74904434250766,
|
|
"logps/rejected": -350.9490315495208,
|
|
"loss": 0.5139,
|
|
"loss/base_kto": 3.793339252471924,
|
|
"loss/total_with_kl": 4.11139440536499,
|
|
"metrics/advantage_var": 191.02430725097656,
|
|
"regularization/advantage_var": 191.02430725097656,
|
|
"regularization/kl": 0.3180554509162903,
|
|
"rewards/chosen": 0.48558973026567276,
|
|
"rewards/margins": 0.1880954574303343,
|
|
"rewards/rejected": 0.29749427283533847,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 29.357450485229492,
|
|
"kl": 11.40846061706543,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37080985.754916795,
|
|
"logits/rejected": -38277980.22617125,
|
|
"logps/chosen": -501.6247163388805,
|
|
"logps/rejected": -382.78412762520196,
|
|
"loss": 0.5205,
|
|
"loss/base_kto": 3.7919883728027344,
|
|
"loss/total_with_kl": 4.163951873779297,
|
|
"metrics/advantage_var": 223.40139770507812,
|
|
"regularization/advantage_var": 223.40139770507812,
|
|
"regularization/kl": 0.3719632923603058,
|
|
"rewards/chosen": 0.22859849799959814,
|
|
"rewards/margins": 0.20049083792479408,
|
|
"rewards/rejected": 0.028107660074804055,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 18.8193359375,
|
|
"kl": 21.178363800048828,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36232604.85079365,
|
|
"logits/rejected": -38612396.50461538,
|
|
"logps/chosen": -493.13412698412696,
|
|
"logps/rejected": -358.96997596153847,
|
|
"loss": 0.5237,
|
|
"loss/base_kto": 3.816561222076416,
|
|
"loss/total_with_kl": 4.189939022064209,
|
|
"metrics/advantage_var": 224.25108337402344,
|
|
"regularization/advantage_var": 224.25108337402344,
|
|
"regularization/kl": 0.3733780086040497,
|
|
"rewards/chosen": 0.36637655591207835,
|
|
"rewards/margins": 0.16838343879969855,
|
|
"rewards/rejected": 0.1979931171123798,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 12.725244522094727,
|
|
"kl": 25.881696701049805,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36246479.83004552,
|
|
"logits/rejected": -38376083.58132045,
|
|
"logps/chosen": -479.6000569044006,
|
|
"logps/rejected": -351.1389895330113,
|
|
"loss": 0.511,
|
|
"loss/base_kto": 3.7016937732696533,
|
|
"loss/total_with_kl": 4.088140487670898,
|
|
"metrics/advantage_var": 232.1000518798828,
|
|
"regularization/advantage_var": 232.1000518798828,
|
|
"regularization/kl": 0.3864465653896332,
|
|
"rewards/chosen": 0.4568905171928348,
|
|
"rewards/margins": 0.2578212345259543,
|
|
"rewards/rejected": 0.19906928266688054,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 131.57684326171875,
|
|
"kl": 29.29970359802246,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36191890.977917984,
|
|
"logits/rejected": -38352350.7120743,
|
|
"logps/chosen": -487.19962539432174,
|
|
"logps/rejected": -384.77087364551085,
|
|
"loss": 0.5272,
|
|
"loss/base_kto": 3.760270833969116,
|
|
"loss/total_with_kl": 4.2179059982299805,
|
|
"metrics/advantage_var": 274.8559265136719,
|
|
"regularization/advantage_var": 274.8559265136719,
|
|
"regularization/kl": 0.4576350748538971,
|
|
"rewards/chosen": 0.43132303039358244,
|
|
"rewards/margins": 0.1845467920790749,
|
|
"rewards/rejected": 0.24677623831450754,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 32.52603530883789,
|
|
"kl": 5.176639080047607,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -34978281.22570016,
|
|
"logits/rejected": -37123606.062407136,
|
|
"logps/chosen": -492.46643327841844,
|
|
"logps/rejected": -384.6894502228826,
|
|
"loss": 0.521,
|
|
"loss/base_kto": 3.815791368484497,
|
|
"loss/total_with_kl": 4.168036937713623,
|
|
"metrics/advantage_var": 211.55894470214844,
|
|
"regularization/advantage_var": 211.55894470214844,
|
|
"regularization/kl": 0.35224565863609314,
|
|
"rewards/chosen": -0.028169433995958608,
|
|
"rewards/margins": 0.1667565855904316,
|
|
"rewards/rejected": -0.19492601958639022,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 15.889501571655273,
|
|
"kl": 2.1030290126800537,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -37257417.549206346,
|
|
"logits/rejected": -38514832.935384616,
|
|
"logps/chosen": -508.6459821428571,
|
|
"logps/rejected": -379.45134615384615,
|
|
"loss": 0.5209,
|
|
"loss/base_kto": 3.7495439052581787,
|
|
"loss/total_with_kl": 4.16711950302124,
|
|
"metrics/advantage_var": 250.7963409423828,
|
|
"regularization/advantage_var": 250.7963409423828,
|
|
"regularization/kl": 0.4175758957862854,
|
|
"rewards/chosen": -0.1384769742451017,
|
|
"rewards/margins": 0.2594093782939608,
|
|
"rewards/rejected": -0.3978863525390625,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 17.3436279296875,
|
|
"kl": 3.039137601852417,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37118635.78947368,
|
|
"logits/rejected": -38403294.47619048,
|
|
"logps/chosen": -487.4664884868421,
|
|
"logps/rejected": -393.27464657738096,
|
|
"loss": 0.5172,
|
|
"loss/base_kto": 3.731966495513916,
|
|
"loss/total_with_kl": 4.137502193450928,
|
|
"metrics/advantage_var": 243.5651397705078,
|
|
"regularization/advantage_var": 243.5651397705078,
|
|
"regularization/kl": 0.4055359959602356,
|
|
"rewards/chosen": -0.0446096376368874,
|
|
"rewards/margins": 0.23779303672020896,
|
|
"rewards/rejected": -0.2824026743570964,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 12.679957389831543,
|
|
"kl": 3.0125131607055664,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -35068480.81012658,
|
|
"logits/rejected": -36613856.39506173,
|
|
"logps/chosen": -497.21004746835445,
|
|
"logps/rejected": -390.15875771604937,
|
|
"loss": 0.5194,
|
|
"loss/base_kto": 3.726527452468872,
|
|
"loss/total_with_kl": 4.15496301651001,
|
|
"metrics/advantage_var": 257.31890869140625,
|
|
"regularization/advantage_var": 257.31890869140625,
|
|
"regularization/kl": 0.42843589186668396,
|
|
"rewards/chosen": -0.045610340335701084,
|
|
"rewards/margins": 0.2788971492807871,
|
|
"rewards/rejected": -0.32450748961648823,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 16.49644660949707,
|
|
"kl": 14.773193359375,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36608568.88888889,
|
|
"logits/rejected": -38326196.40268456,
|
|
"logps/chosen": -503.44590643274853,
|
|
"logps/rejected": -353.9297661493289,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.6660022735595703,
|
|
"loss/total_with_kl": 4.099441051483154,
|
|
"metrics/advantage_var": 260.3235168457031,
|
|
"regularization/advantage_var": 260.3235168457031,
|
|
"regularization/kl": 0.43343862891197205,
|
|
"rewards/chosen": 0.3166549638000845,
|
|
"rewards/margins": 0.2954046038360338,
|
|
"rewards/rejected": 0.021250359964050704,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 18.86716651916504,
|
|
"kl": 24.02515411376953,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36005391.802469134,
|
|
"logits/rejected": -37409529.51898734,
|
|
"logps/chosen": -482.8971354166667,
|
|
"logps/rejected": -348.525761471519,
|
|
"loss": 0.5237,
|
|
"loss/base_kto": 3.7093505859375,
|
|
"loss/total_with_kl": 4.189932823181152,
|
|
"metrics/advantage_var": 288.637451171875,
|
|
"regularization/advantage_var": 288.637451171875,
|
|
"regularization/kl": 0.4805813431739807,
|
|
"rewards/chosen": 0.49618826972113717,
|
|
"rewards/margins": 0.26080010544231,
|
|
"rewards/rejected": 0.23538816427882714,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 18.295459747314453,
|
|
"kl": 10.224114418029785,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36287873.87928222,
|
|
"logits/rejected": -38906473.16341829,
|
|
"logps/chosen": -488.47705954323004,
|
|
"logps/rejected": -365.48046289355324,
|
|
"loss": 0.5156,
|
|
"loss/base_kto": 3.6996846199035645,
|
|
"loss/total_with_kl": 4.1250152587890625,
|
|
"metrics/advantage_var": 255.45396423339844,
|
|
"regularization/advantage_var": 255.45396423339844,
|
|
"regularization/kl": 0.42533078789711,
|
|
"rewards/chosen": 0.17645793871233942,
|
|
"rewards/margins": 0.25755616065849524,
|
|
"rewards/rejected": -0.08109822194615583,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 14.254644393920898,
|
|
"kl": 8.755472183227539,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35747625.6,
|
|
"logits/rejected": -37146771.2,
|
|
"logps/chosen": -495.8552734375,
|
|
"logps/rejected": -380.7497802734375,
|
|
"loss": 0.5209,
|
|
"loss/base_kto": 3.6856186389923096,
|
|
"loss/total_with_kl": 4.167328834533691,
|
|
"metrics/advantage_var": 289.31536865234375,
|
|
"regularization/advantage_var": 289.31536865234375,
|
|
"regularization/kl": 0.4817100465297699,
|
|
"rewards/chosen": 0.1729241967201233,
|
|
"rewards/margins": 0.2861413359642029,
|
|
"rewards/rejected": -0.11321713924407958,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 14.803353309631348,
|
|
"kl": 9.75937557220459,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36956386.86494689,
|
|
"logits/rejected": -37225166.94363929,
|
|
"logps/chosen": -457.41099203338393,
|
|
"logps/rejected": -376.5743508454106,
|
|
"loss": 0.5136,
|
|
"loss/base_kto": 3.7342541217803955,
|
|
"loss/total_with_kl": 4.108650207519531,
|
|
"metrics/advantage_var": 224.862548828125,
|
|
"regularization/advantage_var": 224.862548828125,
|
|
"regularization/kl": 0.3743961453437805,
|
|
"rewards/chosen": 0.15819099206302162,
|
|
"rewards/margins": 0.2578718204585027,
|
|
"rewards/rejected": -0.09968082839548108,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 13.023077964782715,
|
|
"kl": 11.907632827758789,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -37419965.00934579,
|
|
"logits/rejected": -38185871.64890282,
|
|
"logps/chosen": -473.6628699376947,
|
|
"logps/rejected": -384.80721003134795,
|
|
"loss": 0.5177,
|
|
"loss/base_kto": 3.687344789505005,
|
|
"loss/total_with_kl": 4.141505241394043,
|
|
"metrics/advantage_var": 272.76885986328125,
|
|
"regularization/advantage_var": 272.76885986328125,
|
|
"regularization/kl": 0.4541601240634918,
|
|
"rewards/chosen": 0.24721781412760416,
|
|
"rewards/margins": 0.3008286606680014,
|
|
"rewards/rejected": -0.05361084654039724,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 15.86916732788086,
|
|
"kl": 13.91089153289795,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -37737871.72628305,
|
|
"logits/rejected": -38219968.90423862,
|
|
"logps/chosen": -491.63258164852255,
|
|
"logps/rejected": -380.27624607535324,
|
|
"loss": 0.5199,
|
|
"loss/base_kto": 3.6812775135040283,
|
|
"loss/total_with_kl": 4.1595377922058105,
|
|
"metrics/advantage_var": 287.2435302734375,
|
|
"regularization/advantage_var": 287.2435302734375,
|
|
"regularization/kl": 0.47826048731803894,
|
|
"rewards/chosen": 0.3224105479001628,
|
|
"rewards/margins": 0.3177741920913598,
|
|
"rewards/rejected": 0.004636355808803013,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 15.11502456665039,
|
|
"kl": 14.820399284362793,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -37378483.2,
|
|
"logits/rejected": -38810985.6,
|
|
"logps/chosen": -499.506591796875,
|
|
"logps/rejected": -369.033740234375,
|
|
"loss": 0.5161,
|
|
"loss/base_kto": 3.703874349594116,
|
|
"loss/total_with_kl": 4.129049301147461,
|
|
"metrics/advantage_var": 255.36026000976562,
|
|
"regularization/advantage_var": 255.36026000976562,
|
|
"regularization/kl": 0.4251748025417328,
|
|
"rewards/chosen": 0.26812138557434084,
|
|
"rewards/margins": 0.25979379862546925,
|
|
"rewards/rejected": 0.008327586948871613,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 12.838387489318848,
|
|
"kl": 13.0317964553833,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36799193.784911714,
|
|
"logits/rejected": -38216184.98630137,
|
|
"logps/chosen": -494.7509028892456,
|
|
"logps/rejected": -375.41412195585997,
|
|
"loss": 0.5134,
|
|
"loss/base_kto": 3.679405689239502,
|
|
"loss/total_with_kl": 4.107245922088623,
|
|
"metrics/advantage_var": 256.9610290527344,
|
|
"regularization/advantage_var": 256.9610290527344,
|
|
"regularization/kl": 0.42784014344215393,
|
|
"rewards/chosen": 0.17315232964235555,
|
|
"rewards/margins": 0.2958745082945804,
|
|
"rewards/rejected": -0.12272217865222484,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 14.595232963562012,
|
|
"kl": 11.939623832702637,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36126301.769854136,
|
|
"logits/rejected": -36686945.30316742,
|
|
"logps/chosen": -482.40903565640195,
|
|
"logps/rejected": -374.1887254901961,
|
|
"loss": 0.5189,
|
|
"loss/base_kto": 3.674595594406128,
|
|
"loss/total_with_kl": 4.15156888961792,
|
|
"metrics/advantage_var": 286.4701232910156,
|
|
"regularization/advantage_var": 286.4701232910156,
|
|
"regularization/kl": 0.4769727289676666,
|
|
"rewards/chosen": 0.20014795337349323,
|
|
"rewards/margins": 0.30965150234228445,
|
|
"rewards/rejected": -0.10950354896879125,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 19.584970474243164,
|
|
"kl": 4.382344722747803,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36109483.20504732,
|
|
"logits/rejected": -37060931.368421055,
|
|
"logps/chosen": -497.10917783911674,
|
|
"logps/rejected": -363.86595394736844,
|
|
"loss": 0.5239,
|
|
"loss/base_kto": 3.7143712043762207,
|
|
"loss/total_with_kl": 4.19157600402832,
|
|
"metrics/advantage_var": 286.6097106933594,
|
|
"regularization/advantage_var": 286.6097106933594,
|
|
"regularization/kl": 0.4772051274776459,
|
|
"rewards/chosen": -0.04339043650341335,
|
|
"rewards/margins": 0.27319334454704136,
|
|
"rewards/rejected": -0.3165837810504547,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 15.339016914367676,
|
|
"kl": 11.145998001098633,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36344467.986710966,
|
|
"logits/rejected": -37687371.51622419,
|
|
"logps/chosen": -476.24133098006644,
|
|
"logps/rejected": -372.29374078171094,
|
|
"loss": 0.5207,
|
|
"loss/base_kto": 3.6438357830047607,
|
|
"loss/total_with_kl": 4.165812015533447,
|
|
"metrics/advantage_var": 313.4992370605469,
|
|
"regularization/advantage_var": 313.4992370605469,
|
|
"regularization/kl": 0.5219761729240417,
|
|
"rewards/chosen": 0.19266740665879362,
|
|
"rewards/margins": 0.33707507847185825,
|
|
"rewards/rejected": -0.14440767181306463,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 26.200326919555664,
|
|
"kl": 3.715789794921875,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -35384174.60763359,
|
|
"logits/rejected": -35745798.5536,
|
|
"logps/chosen": -458.30143129770994,
|
|
"logps/rejected": -371.50825,
|
|
"loss": 0.5302,
|
|
"loss/base_kto": 3.7983169555664062,
|
|
"loss/total_with_kl": 4.2412495613098145,
|
|
"metrics/advantage_var": 266.0255126953125,
|
|
"regularization/advantage_var": 266.0255126953125,
|
|
"regularization/kl": 0.44293245673179626,
|
|
"rewards/chosen": -0.08463416063148557,
|
|
"rewards/margins": 0.21929518507163945,
|
|
"rewards/rejected": -0.303929345703125,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 17.56812858581543,
|
|
"kl": 18.693069458007812,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35942031.67751938,
|
|
"logits/rejected": -35166940.11968504,
|
|
"logps/chosen": -473.8987403100775,
|
|
"logps/rejected": -373.3375246062992,
|
|
"loss": 0.524,
|
|
"loss/base_kto": 3.657953977584839,
|
|
"loss/total_with_kl": 4.19184684753418,
|
|
"metrics/advantage_var": 320.6563720703125,
|
|
"regularization/advantage_var": 320.6563720703125,
|
|
"regularization/kl": 0.533892810344696,
|
|
"rewards/chosen": 0.3846660288729409,
|
|
"rewards/margins": 0.3189899659961569,
|
|
"rewards/rejected": 0.06567606287678396,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 10.936003684997559,
|
|
"kl": 13.084571838378906,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -34986207.94928685,
|
|
"logits/rejected": -35362913.82434515,
|
|
"logps/chosen": -449.13287440570525,
|
|
"logps/rejected": -334.2512278505393,
|
|
"loss": 0.502,
|
|
"loss/base_kto": 3.5975258350372314,
|
|
"loss/total_with_kl": 4.015763759613037,
|
|
"metrics/advantage_var": 251.19371032714844,
|
|
"regularization/advantage_var": 251.19371032714844,
|
|
"regularization/kl": 0.41823750734329224,
|
|
"rewards/chosen": 0.27444801391021195,
|
|
"rewards/margins": 0.3917814010999389,
|
|
"rewards/rejected": -0.11733338718972698,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 19.385421752929688,
|
|
"kl": 8.8245210647583,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -35239906.602870815,
|
|
"logits/rejected": -36412386.20520674,
|
|
"logps/chosen": -468.02641547049444,
|
|
"logps/rejected": -387.21604613323126,
|
|
"loss": 0.5238,
|
|
"loss/base_kto": 3.7126381397247314,
|
|
"loss/total_with_kl": 4.190079689025879,
|
|
"metrics/advantage_var": 286.7515563964844,
|
|
"regularization/advantage_var": 286.7515563964844,
|
|
"regularization/kl": 0.47744131088256836,
|
|
"rewards/chosen": 0.07935350287283817,
|
|
"rewards/margins": 0.29322699951992376,
|
|
"rewards/rejected": -0.21387349664708558,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 14.663601875305176,
|
|
"kl": 7.2484130859375,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35352050.87179487,
|
|
"logits/rejected": -36537226.165316045,
|
|
"logps/chosen": -494.62877073906486,
|
|
"logps/rejected": -374.1108438006483,
|
|
"loss": 0.5194,
|
|
"loss/base_kto": 3.6950180530548096,
|
|
"loss/total_with_kl": 4.15555477142334,
|
|
"metrics/advantage_var": 276.598388671875,
|
|
"regularization/advantage_var": 276.598388671875,
|
|
"regularization/kl": 0.46053630113601685,
|
|
"rewards/chosen": 0.09303807277305454,
|
|
"rewards/margins": 0.28864844065776,
|
|
"rewards/rejected": -0.19561036788470548,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 16.714582443237305,
|
|
"kl": 17.105743408203125,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -38186708.36802413,
|
|
"logits/rejected": -37993178.243111834,
|
|
"logps/chosen": -506.79520173453994,
|
|
"logps/rejected": -381.8746454619125,
|
|
"loss": 0.5168,
|
|
"loss/base_kto": 3.6692006587982178,
|
|
"loss/total_with_kl": 4.13415002822876,
|
|
"metrics/advantage_var": 279.248779296875,
|
|
"regularization/advantage_var": 279.248779296875,
|
|
"regularization/kl": 0.46494922041893005,
|
|
"rewards/chosen": 0.39226553846448436,
|
|
"rewards/margins": 0.2876216039147477,
|
|
"rewards/rejected": 0.10464393454973663,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 18.708269119262695,
|
|
"kl": 14.641237258911133,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36220029.58490566,
|
|
"logits/rejected": -37642640.69565217,
|
|
"logps/chosen": -494.0291863207547,
|
|
"logps/rejected": -355.74058618012424,
|
|
"loss": 0.5168,
|
|
"loss/base_kto": 3.6611649990081787,
|
|
"loss/total_with_kl": 4.134397029876709,
|
|
"metrics/advantage_var": 284.2236022949219,
|
|
"regularization/advantage_var": 284.2236022949219,
|
|
"regularization/kl": 0.4732322692871094,
|
|
"rewards/chosen": 0.26615013266509435,
|
|
"rewards/margins": 0.30125557729483465,
|
|
"rewards/rejected": -0.03510544462974027,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 26.249277114868164,
|
|
"kl": 13.373329162597656,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35497366.974358976,
|
|
"logits/rejected": -36902468.682926826,
|
|
"logps/chosen": -465.55398637820514,
|
|
"logps/rejected": -383.5540205792683,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.695056200027466,
|
|
"loss/total_with_kl": 4.1215715408325195,
|
|
"metrics/advantage_var": 256.1650390625,
|
|
"regularization/advantage_var": 256.1650390625,
|
|
"regularization/kl": 0.4265148341655731,
|
|
"rewards/chosen": 0.13917385003505608,
|
|
"rewards/margins": 0.2562612607525318,
|
|
"rewards/rejected": -0.1170874107174757,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 12.480875015258789,
|
|
"kl": 14.748756408691406,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37256670.94173228,
|
|
"logits/rejected": -38256262.151937984,
|
|
"logps/chosen": -482.16225393700785,
|
|
"logps/rejected": -385.0124515503876,
|
|
"loss": 0.5089,
|
|
"loss/base_kto": 3.6076576709747314,
|
|
"loss/total_with_kl": 4.0711669921875,
|
|
"metrics/advantage_var": 278.38385009765625,
|
|
"regularization/advantage_var": 278.38385009765625,
|
|
"regularization/kl": 0.46350908279418945,
|
|
"rewards/chosen": 0.29258932278850885,
|
|
"rewards/margins": 0.34477712559858636,
|
|
"rewards/rejected": -0.05218780281007752,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 17.288246154785156,
|
|
"kl": 12.5535888671875,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -38125598.98335855,
|
|
"logits/rejected": -38174147.6187399,
|
|
"logps/chosen": -479.7106183812405,
|
|
"logps/rejected": -382.77867528271406,
|
|
"loss": 0.5207,
|
|
"loss/base_kto": 3.720900058746338,
|
|
"loss/total_with_kl": 4.1653361320495605,
|
|
"metrics/advantage_var": 266.92828369140625,
|
|
"regularization/advantage_var": 266.92828369140625,
|
|
"regularization/kl": 0.44443559646606445,
|
|
"rewards/chosen": 0.24331503448255484,
|
|
"rewards/margins": 0.24921536201406436,
|
|
"rewards/rejected": -0.0059003275315095225,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 27.970935821533203,
|
|
"kl": 19.024961471557617,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -37159446.84958678,
|
|
"logits/rejected": -37783288.03555556,
|
|
"logps/chosen": -461.6587809917355,
|
|
"logps/rejected": -370.68409722222225,
|
|
"loss": 0.5233,
|
|
"loss/base_kto": 3.680556535720825,
|
|
"loss/total_with_kl": 4.186591625213623,
|
|
"metrics/advantage_var": 303.924560546875,
|
|
"regularization/advantage_var": 303.924560546875,
|
|
"regularization/kl": 0.5060344338417053,
|
|
"rewards/chosen": 0.34186113846203514,
|
|
"rewards/margins": 0.24035254154724928,
|
|
"rewards/rejected": 0.10150859691478588,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 12.256890296936035,
|
|
"kl": 11.472315788269043,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36766501.35791091,
|
|
"logits/rejected": -38025525.316375196,
|
|
"logps/chosen": -486.63148041474653,
|
|
"logps/rejected": -384.1498161764706,
|
|
"loss": 0.5195,
|
|
"loss/base_kto": 3.6514556407928467,
|
|
"loss/total_with_kl": 4.156064510345459,
|
|
"metrics/advantage_var": 303.0688171386719,
|
|
"regularization/advantage_var": 303.0688171386719,
|
|
"regularization/kl": 0.5046095252037048,
|
|
"rewards/chosen": 0.2328626688358055,
|
|
"rewards/margins": 0.34065466044109416,
|
|
"rewards/rejected": -0.10779199160528866,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 14.921036720275879,
|
|
"kl": 14.676401138305664,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -37102108.44444445,
|
|
"logits/rejected": -37644236.151898734,
|
|
"logps/chosen": -505.00626929012344,
|
|
"logps/rejected": -377.8958168512658,
|
|
"loss": 0.5201,
|
|
"loss/base_kto": 3.6620795726776123,
|
|
"loss/total_with_kl": 4.160859107971191,
|
|
"metrics/advantage_var": 299.5671691894531,
|
|
"regularization/advantage_var": 299.5671691894531,
|
|
"regularization/kl": 0.498779296875,
|
|
"rewards/chosen": 0.1833903701217086,
|
|
"rewards/margins": 0.2806555231132811,
|
|
"rewards/rejected": -0.09726515299157251,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 16.012065887451172,
|
|
"kl": 18.5018310546875,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35699431.541401275,
|
|
"logits/rejected": -38184702.42944785,
|
|
"logps/chosen": -472.077826433121,
|
|
"logps/rejected": -359.90203220858893,
|
|
"loss": 0.5079,
|
|
"loss/base_kto": 3.6436784267425537,
|
|
"loss/total_with_kl": 4.0634050369262695,
|
|
"metrics/advantage_var": 252.0883026123047,
|
|
"regularization/advantage_var": 252.0883026123047,
|
|
"regularization/kl": 0.41972705721855164,
|
|
"rewards/chosen": 0.2600595753663664,
|
|
"rewards/margins": 0.2945126339017024,
|
|
"rewards/rejected": -0.034453058535335986,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 17.976825714111328,
|
|
"kl": 20.56618309020996,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36451590.717201166,
|
|
"logits/rejected": -37006042.93602694,
|
|
"logps/chosen": -479.79282069970844,
|
|
"logps/rejected": -400.649647516835,
|
|
"loss": 0.512,
|
|
"loss/base_kto": 3.646868944168091,
|
|
"loss/total_with_kl": 4.095690727233887,
|
|
"metrics/advantage_var": 269.56268310546875,
|
|
"regularization/advantage_var": 269.56268310546875,
|
|
"regularization/kl": 0.4488218426704407,
|
|
"rewards/chosen": 0.38110556199321244,
|
|
"rewards/margins": 0.2800363395975435,
|
|
"rewards/rejected": 0.10106922239566893,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 14.430757522583008,
|
|
"kl": 28.526700973510742,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36917991.58153846,
|
|
"logits/rejected": -37864881.73248408,
|
|
"logps/chosen": -476.4955769230769,
|
|
"logps/rejected": -368.49400378184714,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.432401418685913,
|
|
"loss/total_with_kl": 4.121790885925293,
|
|
"metrics/advantage_var": 414.0479431152344,
|
|
"regularization/advantage_var": 414.0479431152344,
|
|
"regularization/kl": 0.6893897652626038,
|
|
"rewards/chosen": 0.655761483999399,
|
|
"rewards/margins": 0.5595810541155758,
|
|
"rewards/rejected": 0.09618042988382328,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 16.027830123901367,
|
|
"kl": 22.440448760986328,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36057283.66878981,
|
|
"logits/rejected": -36913073.47239264,
|
|
"logps/chosen": -463.7797571656051,
|
|
"logps/rejected": -357.4802051380368,
|
|
"loss": 0.5414,
|
|
"loss/base_kto": 3.18515944480896,
|
|
"loss/total_with_kl": 4.3311591148376465,
|
|
"metrics/advantage_var": 688.2881469726562,
|
|
"regularization/advantage_var": 688.2881469726562,
|
|
"regularization/kl": 1.145999789237976,
|
|
"rewards/chosen": 0.7192742414535231,
|
|
"rewards/margins": 0.8778118311527359,
|
|
"rewards/rejected": -0.15853758969921275,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 24.44966697692871,
|
|
"kl": 22.706125259399414,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36837197.20634921,
|
|
"logits/rejected": -38595253.16923077,
|
|
"logps/chosen": -477.5025297619048,
|
|
"logps/rejected": -400.7340384615385,
|
|
"loss": 0.7072,
|
|
"loss/base_kto": 3.1124749183654785,
|
|
"loss/total_with_kl": 5.657690048217773,
|
|
"metrics/advantage_var": 1528.6578369140625,
|
|
"regularization/advantage_var": 1528.6578369140625,
|
|
"regularization/kl": 2.545215368270874,
|
|
"rewards/chosen": 0.7123586987692212,
|
|
"rewards/margins": 1.0149622237842453,
|
|
"rewards/rejected": -0.302603525015024,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 11.723468780517578,
|
|
"kl": 14.782014846801758,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36865033.54037267,
|
|
"logits/rejected": -37690387.320754714,
|
|
"logps/chosen": -483.75296001552795,
|
|
"logps/rejected": -400.18204599056605,
|
|
"loss": 0.521,
|
|
"loss/base_kto": 3.120527982711792,
|
|
"loss/total_with_kl": 4.167871952056885,
|
|
"metrics/advantage_var": 629.0353393554688,
|
|
"regularization/advantage_var": 629.0353393554688,
|
|
"regularization/kl": 1.0473438501358032,
|
|
"rewards/chosen": 0.6224194639217779,
|
|
"rewards/margins": 0.9727715292900464,
|
|
"rewards/rejected": -0.35035206536826846,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 47.848121643066406,
|
|
"kl": 17.3000545501709,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -37203643.43581616,
|
|
"logits/rejected": -37336807.14946071,
|
|
"logps/chosen": -504.6797246434231,
|
|
"logps/rejected": -365.1705989984592,
|
|
"loss": 0.5173,
|
|
"loss/base_kto": 3.144667148590088,
|
|
"loss/total_with_kl": 4.138281345367432,
|
|
"metrics/advantage_var": 596.765380859375,
|
|
"regularization/advantage_var": 596.765380859375,
|
|
"regularization/kl": 0.9936143755912781,
|
|
"rewards/chosen": 0.6061912089255646,
|
|
"rewards/margins": 0.9172739989300185,
|
|
"rewards/rejected": -0.311082790004454,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 17.364564895629883,
|
|
"kl": 16.685171127319336,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34775951.975346684,
|
|
"logits/rejected": -36733768.62123613,
|
|
"logps/chosen": -485.69597457627117,
|
|
"logps/rejected": -381.38975832012676,
|
|
"loss": 0.5397,
|
|
"loss/base_kto": 3.1717493534088135,
|
|
"loss/total_with_kl": 4.317812442779541,
|
|
"metrics/advantage_var": 688.326171875,
|
|
"regularization/advantage_var": 688.326171875,
|
|
"regularization/kl": 1.1460630893707275,
|
|
"rewards/chosen": 0.5993905324597939,
|
|
"rewards/margins": 0.8719567338406007,
|
|
"rewards/rejected": -0.2725662013808067,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 22.1566162109375,
|
|
"kl": 14.815750122070312,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36743694.989214174,
|
|
"logits/rejected": -37517130.24405705,
|
|
"logps/chosen": -480.1051617873652,
|
|
"logps/rejected": -381.6002872424723,
|
|
"loss": 0.5198,
|
|
"loss/base_kto": 3.1873226165771484,
|
|
"loss/total_with_kl": 4.158039093017578,
|
|
"metrics/advantage_var": 583.0128784179688,
|
|
"regularization/advantage_var": 583.0128784179688,
|
|
"regularization/kl": 0.9707164764404297,
|
|
"rewards/chosen": 0.5157440608749037,
|
|
"rewards/margins": 0.8582328772223433,
|
|
"rewards/rejected": -0.3424888163474396,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 111.60948944091797,
|
|
"kl": 15.490511894226074,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34866865.56765163,
|
|
"logits/rejected": -37142437.97802198,
|
|
"logps/chosen": -467.9593701399689,
|
|
"logps/rejected": -369.319613422292,
|
|
"loss": 0.534,
|
|
"loss/base_kto": 3.219972610473633,
|
|
"loss/total_with_kl": 4.2719950675964355,
|
|
"metrics/advantage_var": 631.8453369140625,
|
|
"regularization/advantage_var": 631.8453369140625,
|
|
"regularization/kl": 1.0520225763320923,
|
|
"rewards/chosen": 0.5430646217073289,
|
|
"rewards/margins": 0.815589756310449,
|
|
"rewards/rejected": -0.2725251346031201,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 19.880008697509766,
|
|
"kl": 19.41802406311035,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36163899.5624037,
|
|
"logits/rejected": -37782344.62123613,
|
|
"logps/chosen": -513.7564522342064,
|
|
"logps/rejected": -379.47432151347067,
|
|
"loss": 0.532,
|
|
"loss/base_kto": 3.1227097511291504,
|
|
"loss/total_with_kl": 4.255713939666748,
|
|
"metrics/advantage_var": 680.4828491210938,
|
|
"regularization/advantage_var": 680.4828491210938,
|
|
"regularization/kl": 1.1330039501190186,
|
|
"rewards/chosen": 0.6724691758354199,
|
|
"rewards/margins": 0.9759685941152461,
|
|
"rewards/rejected": -0.3034994182798262,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 29.675657272338867,
|
|
"kl": 19.0522518157959,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35016233.555895865,
|
|
"logits/rejected": -36021631.79585327,
|
|
"logps/chosen": -505.2218606431853,
|
|
"logps/rejected": -373.53162380382776,
|
|
"loss": 0.5315,
|
|
"loss/base_kto": 3.199005365371704,
|
|
"loss/total_with_kl": 4.2517476081848145,
|
|
"metrics/advantage_var": 632.2777709960938,
|
|
"regularization/advantage_var": 632.2777709960938,
|
|
"regularization/kl": 1.0527423620224,
|
|
"rewards/chosen": 0.6828806761760624,
|
|
"rewards/margins": 0.8645740498607402,
|
|
"rewards/rejected": -0.18169337368467778,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 15.505671501159668,
|
|
"kl": 17.601240158081055,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35504928.0,
|
|
"logits/rejected": -36978988.8,
|
|
"logps/chosen": -460.32314453125,
|
|
"logps/rejected": -373.08994140625,
|
|
"loss": 0.5234,
|
|
"loss/base_kto": 3.1828761100769043,
|
|
"loss/total_with_kl": 4.187551975250244,
|
|
"metrics/advantage_var": 603.4091186523438,
|
|
"regularization/advantage_var": 603.4091186523438,
|
|
"regularization/kl": 1.0046762228012085,
|
|
"rewards/chosen": 0.6137538433074952,
|
|
"rewards/margins": 0.8548269987106324,
|
|
"rewards/rejected": -0.2410731554031372,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 20.777111053466797,
|
|
"kl": 16.595083236694336,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35664453.81818182,
|
|
"logits/rejected": -37547754.409638554,
|
|
"logps/chosen": -468.5825892857143,
|
|
"logps/rejected": -367.95336031626505,
|
|
"loss": 0.5056,
|
|
"loss/base_kto": 3.174976110458374,
|
|
"loss/total_with_kl": 4.044863700866699,
|
|
"metrics/advantage_var": 522.4550170898438,
|
|
"regularization/advantage_var": 522.4550170898438,
|
|
"regularization/kl": 0.86988765001297,
|
|
"rewards/chosen": 0.5709245855158026,
|
|
"rewards/margins": 0.8491906190101315,
|
|
"rewards/rejected": -0.2782660334943289,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 21.215492248535156,
|
|
"kl": 15.830972671508789,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34005008.46302251,
|
|
"logits/rejected": -36186712.705167174,
|
|
"logps/chosen": -475.83832395498393,
|
|
"logps/rejected": -367.00113981762917,
|
|
"loss": 0.5125,
|
|
"loss/base_kto": 3.14848256111145,
|
|
"loss/total_with_kl": 4.100399971008301,
|
|
"metrics/advantage_var": 571.72216796875,
|
|
"regularization/advantage_var": 571.72216796875,
|
|
"regularization/kl": 0.9519174695014954,
|
|
"rewards/chosen": 0.5289721718938404,
|
|
"rewards/margins": 0.866860292411745,
|
|
"rewards/rejected": -0.3378881205179046,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 28.345291137695312,
|
|
"kl": 18.425092697143555,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -36062101.30015552,
|
|
"logits/rejected": -36489894.379905805,
|
|
"logps/chosen": -490.91417185069986,
|
|
"logps/rejected": -398.04466738618527,
|
|
"loss": 0.5135,
|
|
"loss/base_kto": 3.1371660232543945,
|
|
"loss/total_with_kl": 4.108198642730713,
|
|
"metrics/advantage_var": 583.2025756835938,
|
|
"regularization/advantage_var": 583.2025756835938,
|
|
"regularization/kl": 0.9710323214530945,
|
|
"rewards/chosen": 0.6313026629932932,
|
|
"rewards/margins": 0.9115406483617194,
|
|
"rewards/rejected": -0.2802379853684262,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 12.197693824768066,
|
|
"kl": 16.49967384338379,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34652210.7244582,
|
|
"logits/rejected": -36630841.337539434,
|
|
"logps/chosen": -485.7217492260062,
|
|
"logps/rejected": -377.4125098580442,
|
|
"loss": 0.5388,
|
|
"loss/base_kto": 3.1774182319641113,
|
|
"loss/total_with_kl": 4.310180187225342,
|
|
"metrics/advantage_var": 680.337158203125,
|
|
"regularization/advantage_var": 680.337158203125,
|
|
"regularization/kl": 1.1327613592147827,
|
|
"rewards/chosen": 0.6493429178054857,
|
|
"rewards/margins": 0.8816621208904374,
|
|
"rewards/rejected": -0.2323192030849517,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 12.110732078552246,
|
|
"kl": 15.026209831237793,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35231660.929797195,
|
|
"logits/rejected": -35182101.633802816,
|
|
"logps/chosen": -473.28207878315135,
|
|
"logps/rejected": -377.75750684663535,
|
|
"loss": 0.4983,
|
|
"loss/base_kto": 3.1897284984588623,
|
|
"loss/total_with_kl": 3.9866340160369873,
|
|
"metrics/advantage_var": 478.62188720703125,
|
|
"regularization/advantage_var": 478.62188720703125,
|
|
"regularization/kl": 0.796905517578125,
|
|
"rewards/chosen": 0.5617964293767063,
|
|
"rewards/margins": 0.8690470955100078,
|
|
"rewards/rejected": -0.30725066613330154,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 15.362382888793945,
|
|
"kl": 18.008405685424805,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -33535168.882082697,
|
|
"logits/rejected": -35901397.537480064,
|
|
"logps/chosen": -457.31537136294025,
|
|
"logps/rejected": -375.2753688197767,
|
|
"loss": 0.507,
|
|
"loss/base_kto": 3.1821868419647217,
|
|
"loss/total_with_kl": 4.056239604949951,
|
|
"metrics/advantage_var": 524.9563598632812,
|
|
"regularization/advantage_var": 524.9563598632812,
|
|
"regularization/kl": 0.8740522265434265,
|
|
"rewards/chosen": 0.6425474672186064,
|
|
"rewards/margins": 0.8705690676285447,
|
|
"rewards/rejected": -0.22802160040993819,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 13.028266906738281,
|
|
"kl": 20.347318649291992,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35240293.58730159,
|
|
"logits/rejected": -36347024.935384616,
|
|
"logps/chosen": -470.25982142857146,
|
|
"logps/rejected": -367.3821394230769,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.178182601928711,
|
|
"loss/total_with_kl": 4.143165111541748,
|
|
"metrics/advantage_var": 579.5691528320312,
|
|
"regularization/advantage_var": 579.5691528320312,
|
|
"regularization/kl": 0.9649826288223267,
|
|
"rewards/chosen": 0.6228441995287698,
|
|
"rewards/margins": 0.8269883692628444,
|
|
"rewards/rejected": -0.2041441697340745,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 13.994969367980957,
|
|
"kl": 12.647303581237793,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34107188.14088821,
|
|
"logits/rejected": -34854216.267942585,
|
|
"logps/chosen": -469.74813361408883,
|
|
"logps/rejected": -374.46511164274324,
|
|
"loss": 0.5208,
|
|
"loss/base_kto": 3.205705404281616,
|
|
"loss/total_with_kl": 4.166758060455322,
|
|
"metrics/advantage_var": 577.208740234375,
|
|
"regularization/advantage_var": 577.208740234375,
|
|
"regularization/kl": 0.9610525369644165,
|
|
"rewards/chosen": 0.5281509469515697,
|
|
"rewards/margins": 0.8834324056587068,
|
|
"rewards/rejected": -0.3552814587071372,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 21.57211685180664,
|
|
"kl": 23.30204963684082,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -36361083.974842764,
|
|
"logits/rejected": -36832236.91925466,
|
|
"logps/chosen": -484.1163522012579,
|
|
"logps/rejected": -376.38327348602485,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.1640663146972656,
|
|
"loss/total_with_kl": 4.1570353507995605,
|
|
"metrics/advantage_var": 596.3777465820312,
|
|
"regularization/advantage_var": 596.3777465820312,
|
|
"regularization/kl": 0.9929689764976501,
|
|
"rewards/chosen": 0.7034506168005601,
|
|
"rewards/margins": 0.8748559470083375,
|
|
"rewards/rejected": -0.17140533020777732,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 14.304961204528809,
|
|
"kl": 11.102791786193848,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -36154552.980645165,
|
|
"logits/rejected": -37462292.16969697,
|
|
"logps/chosen": -498.20650201612904,
|
|
"logps/rejected": -384.77845643939395,
|
|
"loss": 0.5262,
|
|
"loss/base_kto": 3.193181037902832,
|
|
"loss/total_with_kl": 4.209733486175537,
|
|
"metrics/advantage_var": 610.5421142578125,
|
|
"regularization/advantage_var": 610.5421142578125,
|
|
"regularization/kl": 1.0165525674819946,
|
|
"rewards/chosen": 0.3935492730909778,
|
|
"rewards/margins": 0.8816277199127107,
|
|
"rewards/rejected": -0.48807844682173296,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 18.025127410888672,
|
|
"kl": 14.871289253234863,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34359508.73798449,
|
|
"logits/rejected": -36444150.32440945,
|
|
"logps/chosen": -480.859253875969,
|
|
"logps/rejected": -355.9941929133858,
|
|
"loss": 0.5052,
|
|
"loss/base_kto": 3.2017250061035156,
|
|
"loss/total_with_kl": 4.041450500488281,
|
|
"metrics/advantage_var": 504.3397521972656,
|
|
"regularization/advantage_var": 504.3397521972656,
|
|
"regularization/kl": 0.8397256731987,
|
|
"rewards/chosen": 0.5066620109617248,
|
|
"rewards/margins": 0.8373793470607036,
|
|
"rewards/rejected": -0.33071733609897885,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 12.405550003051758,
|
|
"kl": 19.35162925720215,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34966591.209876545,
|
|
"logits/rejected": -36841151.18987342,
|
|
"logps/chosen": -498.6701388888889,
|
|
"logps/rejected": -370.04966871044303,
|
|
"loss": 0.5206,
|
|
"loss/base_kto": 3.1798698902130127,
|
|
"loss/total_with_kl": 4.164729118347168,
|
|
"metrics/advantage_var": 591.5068969726562,
|
|
"regularization/advantage_var": 591.5068969726562,
|
|
"regularization/kl": 0.9848589301109314,
|
|
"rewards/chosen": 0.6469076651114004,
|
|
"rewards/margins": 0.8770311125779007,
|
|
"rewards/rejected": -0.2301234474665002,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 11.384881019592285,
|
|
"kl": 15.16826057434082,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33603148.523882896,
|
|
"logits/rejected": -35084193.87638669,
|
|
"logps/chosen": -460.21398305084745,
|
|
"logps/rejected": -354.7185023771791,
|
|
"loss": 0.507,
|
|
"loss/base_kto": 3.2701079845428467,
|
|
"loss/total_with_kl": 4.056195259094238,
|
|
"metrics/advantage_var": 472.1244201660156,
|
|
"regularization/advantage_var": 472.1244201660156,
|
|
"regularization/kl": 0.786087155342102,
|
|
"rewards/chosen": 0.5577042260779805,
|
|
"rewards/margins": 0.7868489194435461,
|
|
"rewards/rejected": -0.22914469336556556,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 21.23419189453125,
|
|
"kl": 10.736686706542969,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35548129.684210524,
|
|
"logits/rejected": -37481152.0,
|
|
"logps/chosen": -502.76932565789474,
|
|
"logps/rejected": -379.7354445684524,
|
|
"loss": 0.544,
|
|
"loss/base_kto": 3.194667100906372,
|
|
"loss/total_with_kl": 4.352288246154785,
|
|
"metrics/advantage_var": 695.2678833007812,
|
|
"regularization/advantage_var": 695.2678833007812,
|
|
"regularization/kl": 1.157620906829834,
|
|
"rewards/chosen": 0.4874106959292763,
|
|
"rewards/margins": 0.878785374768097,
|
|
"rewards/rejected": -0.3913746788388207,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 12.550798416137695,
|
|
"kl": 15.807379722595215,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34019533.1155624,
|
|
"logits/rejected": -35015321.35657686,
|
|
"logps/chosen": -489.33407164869027,
|
|
"logps/rejected": -366.04796454041207,
|
|
"loss": 0.5074,
|
|
"loss/base_kto": 3.1485722064971924,
|
|
"loss/total_with_kl": 4.059228420257568,
|
|
"metrics/advantage_var": 546.9407348632812,
|
|
"regularization/advantage_var": 546.9407348632812,
|
|
"regularization/kl": 0.9106562733650208,
|
|
"rewards/chosen": 0.5557488910222458,
|
|
"rewards/margins": 0.8697980812956214,
|
|
"rewards/rejected": -0.3140491902733756,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 13.203516960144043,
|
|
"kl": 12.83787727355957,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34067463.91344668,
|
|
"logits/rejected": -34785558.24328594,
|
|
"logps/chosen": -470.4313176197836,
|
|
"logps/rejected": -361.5815560821485,
|
|
"loss": 0.5118,
|
|
"loss/base_kto": 3.2267396450042725,
|
|
"loss/total_with_kl": 4.094069957733154,
|
|
"metrics/advantage_var": 520.9192504882812,
|
|
"regularization/advantage_var": 520.9192504882812,
|
|
"regularization/kl": 0.8673305511474609,
|
|
"rewards/chosen": 0.5369890970645769,
|
|
"rewards/margins": 0.8186976321128342,
|
|
"rewards/rejected": -0.2817085350482573,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 17.600608825683594,
|
|
"kl": 21.662384033203125,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34932157.28257687,
|
|
"logits/rejected": -34804480.42881072,
|
|
"logps/chosen": -470.1298041727672,
|
|
"logps/rejected": -360.8416561976549,
|
|
"loss": 0.5017,
|
|
"loss/base_kto": 3.214611530303955,
|
|
"loss/total_with_kl": 4.013759136199951,
|
|
"metrics/advantage_var": 479.9686584472656,
|
|
"regularization/advantage_var": 479.9686584472656,
|
|
"regularization/kl": 0.7991477251052856,
|
|
"rewards/chosen": 0.6904898290229228,
|
|
"rewards/margins": 0.8291842343789673,
|
|
"rewards/rejected": -0.13869440535604455,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 12.323432922363281,
|
|
"kl": 21.623647689819336,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35091959.427701674,
|
|
"logits/rejected": -37777434.298555374,
|
|
"logps/chosen": -465.1884512937595,
|
|
"logps/rejected": -363.67345505617976,
|
|
"loss": 0.509,
|
|
"loss/base_kto": 3.236440420150757,
|
|
"loss/total_with_kl": 4.072193145751953,
|
|
"metrics/advantage_var": 501.95361328125,
|
|
"regularization/advantage_var": 501.95361328125,
|
|
"regularization/kl": 0.8357527852058411,
|
|
"rewards/chosen": 0.670259658604452,
|
|
"rewards/margins": 0.8174756717214263,
|
|
"rewards/rejected": -0.14721601311697433,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 17.97987937927246,
|
|
"kl": 24.90900230407715,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35157740.895522386,
|
|
"logits/rejected": -36166132.249180324,
|
|
"logps/chosen": -463.6839552238806,
|
|
"logps/rejected": -394.0257172131148,
|
|
"loss": 0.5186,
|
|
"loss/base_kto": 3.186596393585205,
|
|
"loss/total_with_kl": 4.149149417877197,
|
|
"metrics/advantage_var": 578.1098022460938,
|
|
"regularization/advantage_var": 578.1098022460938,
|
|
"regularization/kl": 0.9625527262687683,
|
|
"rewards/chosen": 0.752466457993237,
|
|
"rewards/margins": 0.814937616380468,
|
|
"rewards/rejected": -0.062471158387231045,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 15.260648727416992,
|
|
"kl": 23.88718032836914,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35047574.01273885,
|
|
"logits/rejected": -35695019.19018405,
|
|
"logps/chosen": -453.6136544585987,
|
|
"logps/rejected": -390.8337327453988,
|
|
"loss": 0.5149,
|
|
"loss/base_kto": 3.2465310096740723,
|
|
"loss/total_with_kl": 4.119444847106934,
|
|
"metrics/advantage_var": 524.2727661132812,
|
|
"regularization/advantage_var": 524.2727661132812,
|
|
"regularization/kl": 0.8729141354560852,
|
|
"rewards/chosen": 0.6648632462616939,
|
|
"rewards/margins": 0.8069072200323035,
|
|
"rewards/rejected": -0.14204397377060965,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 40.36940002441406,
|
|
"kl": 24.01422882080078,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -33963539.482496195,
|
|
"logits/rejected": -34700914.23434992,
|
|
"logps/chosen": -498.3009893455099,
|
|
"logps/rejected": -375.87713182182983,
|
|
"loss": 0.5174,
|
|
"loss/base_kto": 3.266451358795166,
|
|
"loss/total_with_kl": 4.139065742492676,
|
|
"metrics/advantage_var": 524.0924072265625,
|
|
"regularization/advantage_var": 524.0924072265625,
|
|
"regularization/kl": 0.8726138472557068,
|
|
"rewards/chosen": 0.629130231372479,
|
|
"rewards/margins": 0.7368029089406973,
|
|
"rewards/rejected": -0.1076726775682183,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 13.748312950134277,
|
|
"kl": 18.63722801208496,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33118719.17552335,
|
|
"logits/rejected": -34697340.30955994,
|
|
"logps/chosen": -478.16928341384863,
|
|
"logps/rejected": -366.45023235963583,
|
|
"loss": 0.4882,
|
|
"loss/base_kto": 3.1683220863342285,
|
|
"loss/total_with_kl": 3.9054718017578125,
|
|
"metrics/advantage_var": 442.7323913574219,
|
|
"regularization/advantage_var": 442.7323913574219,
|
|
"regularization/kl": 0.7371494174003601,
|
|
"rewards/chosen": 0.6168487106544384,
|
|
"rewards/margins": 0.8708910625696271,
|
|
"rewards/rejected": -0.25404235191518876,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 45.252323150634766,
|
|
"kl": 14.570375442504883,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34964176.768992245,
|
|
"logits/rejected": -37252874.88503937,
|
|
"logps/chosen": -503.100484496124,
|
|
"logps/rejected": -387.47586122047244,
|
|
"loss": 0.5094,
|
|
"loss/base_kto": 3.2020890712738037,
|
|
"loss/total_with_kl": 4.074826717376709,
|
|
"metrics/advantage_var": 524.1666870117188,
|
|
"regularization/advantage_var": 524.1666870117188,
|
|
"regularization/kl": 0.8727375268936157,
|
|
"rewards/chosen": 0.5829836823219476,
|
|
"rewards/margins": 0.8559707957339187,
|
|
"rewards/rejected": -0.27298711341197096,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 12.377616882324219,
|
|
"kl": 21.388090133666992,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -32697256.038897894,
|
|
"logits/rejected": -35308440.518853694,
|
|
"logps/chosen": -467.6978322528363,
|
|
"logps/rejected": -364.8227281297134,
|
|
"loss": 0.5111,
|
|
"loss/base_kto": 3.175271987915039,
|
|
"loss/total_with_kl": 4.088471412658691,
|
|
"metrics/advantage_var": 548.4679565429688,
|
|
"regularization/advantage_var": 548.4679565429688,
|
|
"regularization/kl": 0.913199245929718,
|
|
"rewards/chosen": 0.6209790988718091,
|
|
"rewards/margins": 0.7975338521019775,
|
|
"rewards/rejected": -0.17655475323016828,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 17.96770477294922,
|
|
"kl": 14.73278522491455,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -33914628.43464567,
|
|
"logits/rejected": -35950239.553488374,
|
|
"logps/chosen": -454.064468503937,
|
|
"logps/rejected": -363.4874515503876,
|
|
"loss": 0.4983,
|
|
"loss/base_kto": 3.246981143951416,
|
|
"loss/total_with_kl": 3.9862403869628906,
|
|
"metrics/advantage_var": 443.9997253417969,
|
|
"regularization/advantage_var": 443.9997253417969,
|
|
"regularization/kl": 0.7392594218254089,
|
|
"rewards/chosen": 0.5303596556656004,
|
|
"rewards/margins": 0.7817680773816251,
|
|
"rewards/rejected": -0.25140842171602473,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 19.688234329223633,
|
|
"kl": 12.038764953613281,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35928193.21135647,
|
|
"logits/rejected": -36824149.59752322,
|
|
"logps/chosen": -509.84966482649844,
|
|
"logps/rejected": -374.5051518962848,
|
|
"loss": 0.5101,
|
|
"loss/base_kto": 3.1669700145721436,
|
|
"loss/total_with_kl": 4.080795764923096,
|
|
"metrics/advantage_var": 548.8440551757812,
|
|
"regularization/advantage_var": 548.8440551757812,
|
|
"regularization/kl": 0.9138253331184387,
|
|
"rewards/chosen": 0.4717784123465842,
|
|
"rewards/margins": 0.9075726403482289,
|
|
"rewards/rejected": -0.43579422800164475,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 19.103435516357422,
|
|
"kl": 12.448521614074707,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35412740.08945687,
|
|
"logits/rejected": -36733310.04281346,
|
|
"logps/chosen": -493.96350838658145,
|
|
"logps/rejected": -365.94973241590213,
|
|
"loss": 0.5006,
|
|
"loss/base_kto": 3.174381971359253,
|
|
"loss/total_with_kl": 4.0048346519470215,
|
|
"metrics/advantage_var": 498.7702331542969,
|
|
"regularization/advantage_var": 498.7702331542969,
|
|
"regularization/kl": 0.8304523825645447,
|
|
"rewards/chosen": 0.482789060939996,
|
|
"rewards/margins": 0.8583273648868615,
|
|
"rewards/rejected": -0.37553830394686544,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 22.631084442138672,
|
|
"kl": 11.49282455444336,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -36232848.74959613,
|
|
"logits/rejected": -35711846.63237519,
|
|
"logps/chosen": -485.0666397415186,
|
|
"logps/rejected": -403.40260968229956,
|
|
"loss": 0.5091,
|
|
"loss/base_kto": 3.2113113403320312,
|
|
"loss/total_with_kl": 4.073005199432373,
|
|
"metrics/advantage_var": 517.5335083007812,
|
|
"regularization/advantage_var": 517.5335083007812,
|
|
"regularization/kl": 0.8616933226585388,
|
|
"rewards/chosen": 0.4211052569511056,
|
|
"rewards/margins": 0.8206276314606082,
|
|
"rewards/rejected": -0.3995223745095026,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 17.78579330444336,
|
|
"kl": 11.901885032653809,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35878359.0708899,
|
|
"logits/rejected": -35850949.30731707,
|
|
"logps/chosen": -473.42317119155354,
|
|
"logps/rejected": -403.6257113821138,
|
|
"loss": 0.4735,
|
|
"loss/base_kto": 3.16479229927063,
|
|
"loss/total_with_kl": 3.7882308959960938,
|
|
"metrics/advantage_var": 374.4375,
|
|
"regularization/advantage_var": 374.4375,
|
|
"regularization/kl": 0.6234384775161743,
|
|
"rewards/chosen": 0.508323703955741,
|
|
"rewards/margins": 0.8497739191170723,
|
|
"rewards/rejected": -0.3414502151613313,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 14.26706600189209,
|
|
"kl": 15.9487943649292,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35277747.48489666,
|
|
"logits/rejected": -35760879.877112135,
|
|
"logps/chosen": -474.0095886327504,
|
|
"logps/rejected": -370.0624039938556,
|
|
"loss": 0.4641,
|
|
"loss/base_kto": 3.1360933780670166,
|
|
"loss/total_with_kl": 3.713010549545288,
|
|
"metrics/advantage_var": 346.4967956542969,
|
|
"regularization/advantage_var": 346.4967956542969,
|
|
"regularization/kl": 0.5769171714782715,
|
|
"rewards/chosen": 0.6138253265041236,
|
|
"rewards/margins": 0.8728191052309622,
|
|
"rewards/rejected": -0.25899377872683854,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 10.086207389831543,
|
|
"kl": 10.952147483825684,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33841814.78317152,
|
|
"logits/rejected": -35682746.39274924,
|
|
"logps/chosen": -481.45009101941747,
|
|
"logps/rejected": -366.0855126510574,
|
|
"loss": 0.4652,
|
|
"loss/base_kto": 3.165739059448242,
|
|
"loss/total_with_kl": 3.7217299938201904,
|
|
"metrics/advantage_var": 333.9284362792969,
|
|
"regularization/advantage_var": 333.9284362792969,
|
|
"regularization/kl": 0.5559908747673035,
|
|
"rewards/chosen": 0.5068062594021794,
|
|
"rewards/margins": 0.8481513866577142,
|
|
"rewards/rejected": -0.34134512725553484,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 12.579350471496582,
|
|
"kl": 10.133259773254395,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -33835605.333333336,
|
|
"logits/rejected": -35693168.3902439,
|
|
"logps/chosen": -469.95678084935895,
|
|
"logps/rejected": -368.1609422637195,
|
|
"loss": 0.4622,
|
|
"loss/base_kto": 3.183089256286621,
|
|
"loss/total_with_kl": 3.697826385498047,
|
|
"metrics/advantage_var": 309.1514587402344,
|
|
"regularization/advantage_var": 309.1514587402344,
|
|
"regularization/kl": 0.5147371888160706,
|
|
"rewards/chosen": 0.4313532022329477,
|
|
"rewards/margins": 0.8092067120893811,
|
|
"rewards/rejected": -0.3778535098564334,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 10.727472305297852,
|
|
"kl": 8.098077774047852,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33968363.32919255,
|
|
"logits/rejected": -34895025.10691824,
|
|
"logps/chosen": -510.988887810559,
|
|
"logps/rejected": -376.80380306603774,
|
|
"loss": 0.4683,
|
|
"loss/base_kto": 3.1759884357452393,
|
|
"loss/total_with_kl": 3.7461278438568115,
|
|
"metrics/advantage_var": 342.4258728027344,
|
|
"regularization/advantage_var": 342.4258728027344,
|
|
"regularization/kl": 0.5701390504837036,
|
|
"rewards/chosen": 0.4529393356038917,
|
|
"rewards/margins": 0.8565572904578374,
|
|
"rewards/rejected": -0.4036179548539456,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 14.88467025756836,
|
|
"kl": 11.241103172302246,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33316190.60030166,
|
|
"logits/rejected": -34900694.92382496,
|
|
"logps/chosen": -469.4418834841629,
|
|
"logps/rejected": -370.39341065640195,
|
|
"loss": 0.462,
|
|
"loss/base_kto": 3.16162109375,
|
|
"loss/total_with_kl": 3.6957168579101562,
|
|
"metrics/advantage_var": 320.778076171875,
|
|
"regularization/advantage_var": 320.778076171875,
|
|
"regularization/kl": 0.5340954661369324,
|
|
"rewards/chosen": 0.5090312698847568,
|
|
"rewards/margins": 0.8521236828488218,
|
|
"rewards/rejected": -0.343092412964065,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 25.685148239135742,
|
|
"kl": 16.984634399414062,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33908816.125195615,
|
|
"logits/rejected": -35897938.271450855,
|
|
"logps/chosen": -493.41867175273865,
|
|
"logps/rejected": -405.77610666926677,
|
|
"loss": 0.4663,
|
|
"loss/base_kto": 3.0821049213409424,
|
|
"loss/total_with_kl": 3.730085849761963,
|
|
"metrics/advantage_var": 389.1777648925781,
|
|
"regularization/advantage_var": 389.1777648925781,
|
|
"regularization/kl": 0.6479809880256653,
|
|
"rewards/chosen": 0.6098842471604803,
|
|
"rewards/margins": 0.9221466052085132,
|
|
"rewards/rejected": -0.3122623580480329,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.96446990966797,
|
|
"kl": 12.134880065917969,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34742538.71627907,
|
|
"logits/rejected": -35549590.37480315,
|
|
"logps/chosen": -456.46143410852716,
|
|
"logps/rejected": -401.46407480314963,
|
|
"loss": 0.4669,
|
|
"loss/base_kto": 3.198035955429077,
|
|
"loss/total_with_kl": 3.735119342803955,
|
|
"metrics/advantage_var": 322.5724792480469,
|
|
"regularization/advantage_var": 322.5724792480469,
|
|
"regularization/kl": 0.5370831489562988,
|
|
"rewards/chosen": 0.49280371850775195,
|
|
"rewards/margins": 0.8004921056801806,
|
|
"rewards/rejected": -0.30768838717242863,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 11.199472427368164,
|
|
"kl": 20.47431755065918,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34062554.96855346,
|
|
"logits/rejected": -35988575.403726704,
|
|
"logps/chosen": -484.41627358490564,
|
|
"logps/rejected": -367.4661296583851,
|
|
"loss": 0.4608,
|
|
"loss/base_kto": 3.1300294399261475,
|
|
"loss/total_with_kl": 3.6866562366485596,
|
|
"metrics/advantage_var": 334.3104553222656,
|
|
"regularization/advantage_var": 334.3104553222656,
|
|
"regularization/kl": 0.5566269755363464,
|
|
"rewards/chosen": 0.6354286145864043,
|
|
"rewards/margins": 0.8338579828839101,
|
|
"rewards/rejected": -0.19842936829750582,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 12.289140701293945,
|
|
"kl": 15.893994331359863,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34612028.87559055,
|
|
"logits/rejected": -34521475.37364341,
|
|
"logps/chosen": -476.50994094488186,
|
|
"logps/rejected": -376.8841569767442,
|
|
"loss": 0.4724,
|
|
"loss/base_kto": 3.167349338531494,
|
|
"loss/total_with_kl": 3.779182195663452,
|
|
"metrics/advantage_var": 367.46722412109375,
|
|
"regularization/advantage_var": 367.46722412109375,
|
|
"regularization/kl": 0.6118329167366028,
|
|
"rewards/chosen": 0.582288558839813,
|
|
"rewards/margins": 0.8153732803680955,
|
|
"rewards/rejected": -0.23308472152828247,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 11.060856819152832,
|
|
"kl": 16.986530303955078,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35904337.32707355,
|
|
"logits/rejected": -35920271.37597504,
|
|
"logps/chosen": -478.1259780907668,
|
|
"logps/rejected": -371.7021255850234,
|
|
"loss": 0.4674,
|
|
"loss/base_kto": 3.1196110248565674,
|
|
"loss/total_with_kl": 3.7393479347229004,
|
|
"metrics/advantage_var": 372.21417236328125,
|
|
"regularization/advantage_var": 372.21417236328125,
|
|
"regularization/kl": 0.6197366118431091,
|
|
"rewards/chosen": 0.6117885153804773,
|
|
"rewards/margins": 0.878207305677011,
|
|
"rewards/rejected": -0.26641879029653376,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 8.367744445800781,
|
|
"kl": 17.850460052490234,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34448627.95751138,
|
|
"logits/rejected": -35959338.04830918,
|
|
"logps/chosen": -467.9199070561457,
|
|
"logps/rejected": -360.73362016908214,
|
|
"loss": 0.4673,
|
|
"loss/base_kto": 3.1459972858428955,
|
|
"loss/total_with_kl": 3.738208055496216,
|
|
"metrics/advantage_var": 355.6819763183594,
|
|
"regularization/advantage_var": 355.6819763183594,
|
|
"regularization/kl": 0.5922104716300964,
|
|
"rewards/chosen": 0.6343060552802542,
|
|
"rewards/margins": 0.84190760559288,
|
|
"rewards/rejected": -0.2076015503126258,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 14.395444869995117,
|
|
"kl": 19.718313217163086,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35564519.26570048,
|
|
"logits/rejected": -37770583.40515933,
|
|
"logps/chosen": -494.33031400966183,
|
|
"logps/rejected": -385.8139226100152,
|
|
"loss": 0.4693,
|
|
"loss/base_kto": 3.1141934394836426,
|
|
"loss/total_with_kl": 3.7542195320129395,
|
|
"metrics/advantage_var": 384.3999938964844,
|
|
"regularization/advantage_var": 384.3999938964844,
|
|
"regularization/kl": 0.6400260329246521,
|
|
"rewards/chosen": 0.6306573004730274,
|
|
"rewards/margins": 0.8881337366422943,
|
|
"rewards/rejected": -0.2574764361692669,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 14.89570426940918,
|
|
"kl": 12.683195114135742,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34974333.58490566,
|
|
"logits/rejected": -36554720.19875777,
|
|
"logps/chosen": -474.2201257861635,
|
|
"logps/rejected": -371.3301387810559,
|
|
"loss": 0.4688,
|
|
"loss/base_kto": 3.1612491607666016,
|
|
"loss/total_with_kl": 3.75059175491333,
|
|
"metrics/advantage_var": 353.959228515625,
|
|
"regularization/advantage_var": 353.959228515625,
|
|
"regularization/kl": 0.5893421173095703,
|
|
"rewards/chosen": 0.5535465456404776,
|
|
"rewards/margins": 0.8291624036355523,
|
|
"rewards/rejected": -0.2756158579950747,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 14.248080253601074,
|
|
"kl": 17.36821937561035,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -36378132.09419152,
|
|
"logits/rejected": -36636722.96111975,
|
|
"logps/chosen": -488.78973704866564,
|
|
"logps/rejected": -388.22278382581646,
|
|
"loss": 0.4662,
|
|
"loss/base_kto": 3.1176862716674805,
|
|
"loss/total_with_kl": 3.72993540763855,
|
|
"metrics/advantage_var": 367.71728515625,
|
|
"regularization/advantage_var": 367.71728515625,
|
|
"regularization/kl": 0.6122493147850037,
|
|
"rewards/chosen": 0.6243669399099784,
|
|
"rewards/margins": 0.8787355573222548,
|
|
"rewards/rejected": -0.25436861741227645,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 12.850327491760254,
|
|
"kl": 16.18716049194336,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34844874.27160494,
|
|
"logits/rejected": -36962329.92405064,
|
|
"logps/chosen": -471.06240354938274,
|
|
"logps/rejected": -345.65758504746833,
|
|
"loss": 0.4568,
|
|
"loss/base_kto": 3.0813486576080322,
|
|
"loss/total_with_kl": 3.6540393829345703,
|
|
"metrics/advantage_var": 343.9584045410156,
|
|
"regularization/advantage_var": 343.9584045410156,
|
|
"regularization/kl": 0.5726906657218933,
|
|
"rewards/chosen": 0.6342455075110918,
|
|
"rewards/margins": 0.9026876372981023,
|
|
"rewards/rejected": -0.2684421297870105,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 13.700258255004883,
|
|
"kl": 19.546560287475586,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34399608.28915662,
|
|
"logits/rejected": -36052626.28571428,
|
|
"logps/chosen": -482.5048004518072,
|
|
"logps/rejected": -387.1890726461039,
|
|
"loss": 0.4657,
|
|
"loss/base_kto": 3.0845305919647217,
|
|
"loss/total_with_kl": 3.7255101203918457,
|
|
"metrics/advantage_var": 384.9725036621094,
|
|
"regularization/advantage_var": 384.9725036621094,
|
|
"regularization/kl": 0.6409791111946106,
|
|
"rewards/chosen": 0.7085653557834855,
|
|
"rewards/margins": 0.9112956336995391,
|
|
"rewards/rejected": -0.20273027791605366,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 11.574172973632812,
|
|
"kl": 20.4260196685791,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34066046.00623053,
|
|
"logits/rejected": -35182123.335423194,
|
|
"logps/chosen": -476.22322819314644,
|
|
"logps/rejected": -360.31279388714734,
|
|
"loss": 0.4615,
|
|
"loss/base_kto": 3.1729626655578613,
|
|
"loss/total_with_kl": 3.691901922225952,
|
|
"metrics/advantage_var": 311.6752624511719,
|
|
"regularization/advantage_var": 311.6752624511719,
|
|
"regularization/kl": 0.5189393162727356,
|
|
"rewards/chosen": 0.6363134651540596,
|
|
"rewards/margins": 0.8237576677326133,
|
|
"rewards/rejected": -0.1874442025785536,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 11.468894004821777,
|
|
"kl": 21.85495376586914,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34670083.30322581,
|
|
"logits/rejected": -36920363.442424245,
|
|
"logps/chosen": -492.3671875,
|
|
"logps/rejected": -373.9569602272727,
|
|
"loss": 0.4618,
|
|
"loss/base_kto": 3.12001895904541,
|
|
"loss/total_with_kl": 3.6945621967315674,
|
|
"metrics/advantage_var": 345.07086181640625,
|
|
"regularization/advantage_var": 345.07086181640625,
|
|
"regularization/kl": 0.5745429396629333,
|
|
"rewards/chosen": 0.6417865384009577,
|
|
"rewards/margins": 0.8691116310634222,
|
|
"rewards/rejected": -0.22732509266246448,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 12.923368453979492,
|
|
"kl": 14.473068237304688,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34439092.497695856,
|
|
"logits/rejected": -36047349.418124005,
|
|
"logps/chosen": -490.56451612903226,
|
|
"logps/rejected": -385.7020319952305,
|
|
"loss": 0.4654,
|
|
"loss/base_kto": 3.133051633834839,
|
|
"loss/total_with_kl": 3.722947835922241,
|
|
"metrics/advantage_var": 354.2922058105469,
|
|
"regularization/advantage_var": 354.2922058105469,
|
|
"regularization/kl": 0.5898965001106262,
|
|
"rewards/chosen": 0.5951190251176075,
|
|
"rewards/margins": 0.8729170940390105,
|
|
"rewards/rejected": -0.277798068921403,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 11.678525924682617,
|
|
"kl": 20.182462692260742,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33799696.66976744,
|
|
"logits/rejected": -37651702.72755905,
|
|
"logps/chosen": -492.54588178294574,
|
|
"logps/rejected": -350.1196850393701,
|
|
"loss": 0.4618,
|
|
"loss/base_kto": 3.1142308712005615,
|
|
"loss/total_with_kl": 3.6947124004364014,
|
|
"metrics/advantage_var": 348.6377258300781,
|
|
"regularization/advantage_var": 348.6377258300781,
|
|
"regularization/kl": 0.5804818272590637,
|
|
"rewards/chosen": 0.6593832326489826,
|
|
"rewards/margins": 0.8821094808419575,
|
|
"rewards/rejected": -0.2227262481929749,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 12.077221870422363,
|
|
"kl": 19.26957130432129,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -33815259.19224556,
|
|
"logits/rejected": -36040109.11951589,
|
|
"logps/chosen": -460.8996365105008,
|
|
"logps/rejected": -371.63178422844175,
|
|
"loss": 0.4616,
|
|
"loss/base_kto": 3.1193220615386963,
|
|
"loss/total_with_kl": 3.6931564807891846,
|
|
"metrics/advantage_var": 344.6451721191406,
|
|
"regularization/advantage_var": 344.6451721191406,
|
|
"regularization/kl": 0.573834240436554,
|
|
"rewards/chosen": 0.6056515596602383,
|
|
"rewards/margins": 0.8724790953616495,
|
|
"rewards/rejected": -0.2668275357014112,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.597661018371582,
|
|
"kl": 17.388717651367188,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34505823.327327326,
|
|
"logits/rejected": -35755911.921824105,
|
|
"logps/chosen": -474.30808933933935,
|
|
"logps/rejected": -387.89418770358304,
|
|
"loss": 0.4602,
|
|
"loss/base_kto": 3.1345348358154297,
|
|
"loss/total_with_kl": 3.681957721710205,
|
|
"metrics/advantage_var": 328.78240966796875,
|
|
"regularization/advantage_var": 328.78240966796875,
|
|
"regularization/kl": 0.5474227070808411,
|
|
"rewards/chosen": 0.6305892531936234,
|
|
"rewards/margins": 0.8648866421215677,
|
|
"rewards/rejected": -0.23429738892794433,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 12.27588176727295,
|
|
"kl": 17.52077865600586,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35681623.56792144,
|
|
"logits/rejected": -36064839.174887896,
|
|
"logps/chosen": -487.5958981178396,
|
|
"logps/rejected": -368.54386210762334,
|
|
"loss": 0.4634,
|
|
"loss/base_kto": 3.1000630855560303,
|
|
"loss/total_with_kl": 3.7071731090545654,
|
|
"metrics/advantage_var": 364.6307067871094,
|
|
"regularization/advantage_var": 364.6307067871094,
|
|
"regularization/kl": 0.6071101427078247,
|
|
"rewards/chosen": 0.6279889245665405,
|
|
"rewards/margins": 0.8973022179323384,
|
|
"rewards/rejected": -0.26931329336579785,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 13.711464881896973,
|
|
"kl": 18.362272262573242,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35292941.59635812,
|
|
"logits/rejected": -35814583.03381643,
|
|
"logps/chosen": -483.1783004552352,
|
|
"logps/rejected": -385.9627616747182,
|
|
"loss": 0.4625,
|
|
"loss/base_kto": 3.1211092472076416,
|
|
"loss/total_with_kl": 3.7002124786376953,
|
|
"metrics/advantage_var": 347.8098449707031,
|
|
"regularization/advantage_var": 347.8098449707031,
|
|
"regularization/kl": 0.579103410243988,
|
|
"rewards/chosen": 0.6044007273834645,
|
|
"rewards/margins": 0.8793050614102987,
|
|
"rewards/rejected": -0.27490433402683423,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 10.548821449279785,
|
|
"kl": 19.548810958862305,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34212640.14883721,
|
|
"logits/rejected": -35593936.83149606,
|
|
"logps/chosen": -481.8520833333333,
|
|
"logps/rejected": -374.74562007874016,
|
|
"loss": 0.4607,
|
|
"loss/base_kto": 3.057556629180908,
|
|
"loss/total_with_kl": 3.685317277908325,
|
|
"metrics/advantage_var": 377.0332946777344,
|
|
"regularization/advantage_var": 377.0332946777344,
|
|
"regularization/kl": 0.6277604103088379,
|
|
"rewards/chosen": 0.6706285580184108,
|
|
"rewards/margins": 0.9070045259302587,
|
|
"rewards/rejected": -0.23637596791184792,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 7.823846817016602,
|
|
"kl": 19.428497314453125,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -36310088.33908948,
|
|
"logits/rejected": -37475453.810264386,
|
|
"logps/chosen": -484.6997154631083,
|
|
"logps/rejected": -369.9785672628305,
|
|
"loss": 0.4609,
|
|
"loss/base_kto": 3.1207172870635986,
|
|
"loss/total_with_kl": 3.687396287918091,
|
|
"metrics/advantage_var": 340.3479919433594,
|
|
"regularization/advantage_var": 340.3479919433594,
|
|
"regularization/kl": 0.5666794776916504,
|
|
"rewards/chosen": 0.6668884181527178,
|
|
"rewards/margins": 0.8729672396938944,
|
|
"rewards/rejected": -0.20607882154117663,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 8.664216995239258,
|
|
"kl": 17.887357711791992,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33848489.07165109,
|
|
"logits/rejected": -36312625.75548589,
|
|
"logps/chosen": -459.8963200934579,
|
|
"logps/rejected": -375.4141849529781,
|
|
"loss": 0.4623,
|
|
"loss/base_kto": 3.104853630065918,
|
|
"loss/total_with_kl": 3.6987578868865967,
|
|
"metrics/advantage_var": 356.699462890625,
|
|
"regularization/advantage_var": 356.699462890625,
|
|
"regularization/kl": 0.5939046144485474,
|
|
"rewards/chosen": 0.6817976811964563,
|
|
"rewards/margins": 0.8805802402953249,
|
|
"rewards/rejected": -0.19878255909886852,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 12.456644058227539,
|
|
"kl": 19.308988571166992,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34445936.918429,
|
|
"logits/rejected": -35370073.475728154,
|
|
"logps/chosen": -476.6726774924471,
|
|
"logps/rejected": -351.433277710356,
|
|
"loss": 0.4636,
|
|
"loss/base_kto": 3.1250016689300537,
|
|
"loss/total_with_kl": 3.708803653717041,
|
|
"metrics/advantage_var": 350.63189697265625,
|
|
"regularization/advantage_var": 350.63189697265625,
|
|
"regularization/kl": 0.5838021636009216,
|
|
"rewards/chosen": 0.6458656795075293,
|
|
"rewards/margins": 0.8593024852079363,
|
|
"rewards/rejected": -0.21343680570040707,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.652802467346191,
|
|
"kl": 19.432348251342773,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34813539.1232,
|
|
"logits/rejected": -35671157.2519084,
|
|
"logps/chosen": -477.397,
|
|
"logps/rejected": -384.62273377862596,
|
|
"loss": 0.4592,
|
|
"loss/base_kto": 3.1075828075408936,
|
|
"loss/total_with_kl": 3.673489809036255,
|
|
"metrics/advantage_var": 339.88409423828125,
|
|
"regularization/advantage_var": 339.88409423828125,
|
|
"regularization/kl": 0.5659070014953613,
|
|
"rewards/chosen": 0.66147333984375,
|
|
"rewards/margins": 0.8755134292253102,
|
|
"rewards/rejected": -0.2140400893815601,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 12.318232536315918,
|
|
"kl": 20.481626510620117,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -36536352.768,
|
|
"logits/rejected": -36707848.59847328,
|
|
"logps/chosen": -503.8237,
|
|
"logps/rejected": -384.4250238549618,
|
|
"loss": 0.4766,
|
|
"loss/base_kto": 3.09419846534729,
|
|
"loss/total_with_kl": 3.8124756813049316,
|
|
"metrics/advantage_var": 431.39794921875,
|
|
"regularization/advantage_var": 431.39794921875,
|
|
"regularization/kl": 0.7182775735855103,
|
|
"rewards/chosen": 0.67184501953125,
|
|
"rewards/margins": 0.9049889963397543,
|
|
"rewards/rejected": -0.2331439768085043,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 14.077258110046387,
|
|
"kl": 20.919355392456055,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35301458.71405493,
|
|
"logits/rejected": -36259285.397882,
|
|
"logps/chosen": -479.1207592891761,
|
|
"logps/rejected": -378.84084247352496,
|
|
"loss": 0.4554,
|
|
"loss/base_kto": 3.1235177516937256,
|
|
"loss/total_with_kl": 3.64288330078125,
|
|
"metrics/advantage_var": 311.9313659667969,
|
|
"regularization/advantage_var": 311.9313659667969,
|
|
"regularization/kl": 0.5193657279014587,
|
|
"rewards/chosen": 0.6403917056870961,
|
|
"rewards/margins": 0.8508738658773264,
|
|
"rewards/rejected": -0.21048216019023025,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 10.713716506958008,
|
|
"kl": 17.51523780822754,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35305854.627871364,
|
|
"logits/rejected": -35604692.31259968,
|
|
"logps/chosen": -488.16653905053596,
|
|
"logps/rejected": -394.6655452551834,
|
|
"loss": 0.4643,
|
|
"loss/base_kto": 3.073132038116455,
|
|
"loss/total_with_kl": 3.714580535888672,
|
|
"metrics/advantage_var": 385.254638671875,
|
|
"regularization/advantage_var": 385.254638671875,
|
|
"regularization/kl": 0.641448974609375,
|
|
"rewards/chosen": 0.6772497390349828,
|
|
"rewards/margins": 0.9285689550437547,
|
|
"rewards/rejected": -0.25131921600877194,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 9.375003814697266,
|
|
"kl": 19.961950302124023,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34793932.55502392,
|
|
"logits/rejected": -34593166.3093415,
|
|
"logps/chosen": -466.6762360446571,
|
|
"logps/rejected": -366.905915007657,
|
|
"loss": 0.4479,
|
|
"loss/base_kto": 3.0823726654052734,
|
|
"loss/total_with_kl": 3.5829293727874756,
|
|
"metrics/advantage_var": 300.634521484375,
|
|
"regularization/advantage_var": 300.634521484375,
|
|
"regularization/kl": 0.500556468963623,
|
|
"rewards/chosen": 0.6577253569826556,
|
|
"rewards/margins": 0.8845860226004771,
|
|
"rewards/rejected": -0.2268606656178216,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 8.692347526550293,
|
|
"kl": 16.773324966430664,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36092797.67272727,
|
|
"logits/rejected": -35963418.425806455,
|
|
"logps/chosen": -482.5468276515152,
|
|
"logps/rejected": -370.9232610887097,
|
|
"loss": 0.4619,
|
|
"loss/base_kto": 3.1150739192962646,
|
|
"loss/total_with_kl": 3.6949822902679443,
|
|
"metrics/advantage_var": 348.2931823730469,
|
|
"regularization/advantage_var": 348.2931823730469,
|
|
"regularization/kl": 0.5799080729484558,
|
|
"rewards/chosen": 0.6352346709280303,
|
|
"rewards/margins": 0.8743967960540384,
|
|
"rewards/rejected": -0.23916212512600807,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 16.19487762451172,
|
|
"kl": 16.275922775268555,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34692276.327044025,
|
|
"logits/rejected": -34984737.39130435,
|
|
"logps/chosen": -477.4905169025157,
|
|
"logps/rejected": -376.2428425854037,
|
|
"loss": 0.4632,
|
|
"loss/base_kto": 3.1360573768615723,
|
|
"loss/total_with_kl": 3.705369710922241,
|
|
"metrics/advantage_var": 341.9292297363281,
|
|
"regularization/advantage_var": 341.9292297363281,
|
|
"regularization/kl": 0.5693122148513794,
|
|
"rewards/chosen": 0.6395522303551248,
|
|
"rewards/margins": 0.8553214116955024,
|
|
"rewards/rejected": -0.21576918134037754,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 13.71744441986084,
|
|
"kl": 17.854293823242188,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34928216.27586207,
|
|
"logits/rejected": -35329256.872274145,
|
|
"logps/chosen": -484.6248040752351,
|
|
"logps/rejected": -353.38101148753896,
|
|
"loss": 0.4618,
|
|
"loss/base_kto": 3.1671254634857178,
|
|
"loss/total_with_kl": 3.6941890716552734,
|
|
"metrics/advantage_var": 316.5549011230469,
|
|
"regularization/advantage_var": 316.5549011230469,
|
|
"regularization/kl": 0.5270638465881348,
|
|
"rewards/chosen": 0.6265961937022433,
|
|
"rewards/margins": 0.8139161542518563,
|
|
"rewards/rejected": -0.18731996054961303,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 10.524263381958008,
|
|
"kl": 17.332983016967773,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34855024.88188976,
|
|
"logits/rejected": -36169356.50232558,
|
|
"logps/chosen": -484.57942913385824,
|
|
"logps/rejected": -380.99890988372096,
|
|
"loss": 0.4618,
|
|
"loss/base_kto": 3.0859153270721436,
|
|
"loss/total_with_kl": 3.6943321228027344,
|
|
"metrics/advantage_var": 365.4156188964844,
|
|
"regularization/advantage_var": 365.4156188964844,
|
|
"regularization/kl": 0.6084169745445251,
|
|
"rewards/chosen": 0.6552281657541831,
|
|
"rewards/margins": 0.9204792426225212,
|
|
"rewards/rejected": -0.26525107686833815,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_loss": 0.5011991713331153,
|
|
"train_runtime": 10985.9282,
|
|
"train_samples_per_second": 13.492,
|
|
"train_steps_per_second": 0.211
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|