f9ac9c0723
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed0 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 12.6637544631958,
|
|
"kl": 10.180965423583984,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36421354.633070864,
|
|
"logits/rejected": -37211979.01395349,
|
|
"logps/chosen": -466.1063976377953,
|
|
"logps/rejected": -375.1452277131783,
|
|
"loss": 0.5341,
|
|
"loss/base_kto": 3.8808181285858154,
|
|
"loss/total_with_kl": 4.272925853729248,
|
|
"metrics/advantage_var": 235.49986267089844,
|
|
"regularization/advantage_var": 235.49986267089844,
|
|
"regularization/kl": 0.3921072483062744,
|
|
"rewards/chosen": 0.14924275555948574,
|
|
"rewards/margins": 0.12256888414210747,
|
|
"rewards/rejected": 0.02667387141737827,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 13.841927528381348,
|
|
"kl": 13.496167182922363,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36202256.17257319,
|
|
"logits/rejected": -37973142.92234548,
|
|
"logps/chosen": -483.42989214175657,
|
|
"logps/rejected": -375.892779318542,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.8225972652435303,
|
|
"loss/total_with_kl": 4.179236888885498,
|
|
"metrics/advantage_var": 214.1981964111328,
|
|
"regularization/advantage_var": 214.1981964111328,
|
|
"regularization/kl": 0.3566400110721588,
|
|
"rewards/chosen": 0.2414894985675077,
|
|
"rewards/margins": 0.16430082235910834,
|
|
"rewards/rejected": 0.07718867620839936,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 14.648808479309082,
|
|
"kl": 7.37127161026001,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35551746.38139535,
|
|
"logits/rejected": -36187295.647244096,
|
|
"logps/chosen": -474.4172965116279,
|
|
"logps/rejected": -359.2525590551181,
|
|
"loss": 0.5207,
|
|
"loss/base_kto": 3.836839437484741,
|
|
"loss/total_with_kl": 4.165544509887695,
|
|
"metrics/advantage_var": 197.42030334472656,
|
|
"regularization/advantage_var": 197.42030334472656,
|
|
"regularization/kl": 0.3287048041820526,
|
|
"rewards/chosen": 0.09676950144213299,
|
|
"rewards/margins": 0.1661965110847265,
|
|
"rewards/rejected": -0.0694270096425935,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 15.017376899719238,
|
|
"kl": 20.64790916442871,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37944748.10413476,
|
|
"logits/rejected": -38923308.09569378,
|
|
"logps/chosen": -489.1542879019908,
|
|
"logps/rejected": -375.0865231259968,
|
|
"loss": 0.514,
|
|
"loss/base_kto": 3.8288886547088623,
|
|
"loss/total_with_kl": 4.112159252166748,
|
|
"metrics/advantage_var": 170.13243103027344,
|
|
"regularization/advantage_var": 170.13243103027344,
|
|
"regularization/kl": 0.2832705080509186,
|
|
"rewards/chosen": 0.3485625940296588,
|
|
"rewards/margins": 0.1400666949691524,
|
|
"rewards/rejected": 0.2084958990605064,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 15.719074249267578,
|
|
"kl": 5.514427185058594,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36266418.943143815,
|
|
"logits/rejected": -37392894.498533726,
|
|
"logps/chosen": -508.6888586956522,
|
|
"logps/rejected": -384.0126008064516,
|
|
"loss": 0.5162,
|
|
"loss/base_kto": 3.780423879623413,
|
|
"loss/total_with_kl": 4.129902362823486,
|
|
"metrics/advantage_var": 209.8968505859375,
|
|
"regularization/advantage_var": 209.8968505859375,
|
|
"regularization/kl": 0.34947827458381653,
|
|
"rewards/chosen": -0.019458957340406334,
|
|
"rewards/margins": 0.20128558117653603,
|
|
"rewards/rejected": -0.22074453851694237,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 9.78908634185791,
|
|
"kl": 8.92809009552002,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35272252.09389672,
|
|
"logits/rejected": -37022721.5975039,
|
|
"logps/chosen": -455.26095461658844,
|
|
"logps/rejected": -359.0306893525741,
|
|
"loss": 0.5268,
|
|
"loss/base_kto": 3.832242250442505,
|
|
"loss/total_with_kl": 4.21449613571167,
|
|
"metrics/advantage_var": 229.5818328857422,
|
|
"regularization/advantage_var": 229.5818328857422,
|
|
"regularization/kl": 0.3822537362575531,
|
|
"rewards/chosen": 0.09355565938218285,
|
|
"rewards/margins": 0.1492870414694976,
|
|
"rewards/rejected": -0.05573138208731474,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 12.168562889099121,
|
|
"kl": 6.102738380432129,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -37019625.73913044,
|
|
"logits/rejected": -37602780.57861635,
|
|
"logps/chosen": -499.48369565217394,
|
|
"logps/rejected": -381.3736242138365,
|
|
"loss": 0.5241,
|
|
"loss/base_kto": 3.7787773609161377,
|
|
"loss/total_with_kl": 4.192605495452881,
|
|
"metrics/advantage_var": 248.54541015625,
|
|
"regularization/advantage_var": 248.54541015625,
|
|
"regularization/kl": 0.4138281047344208,
|
|
"rewards/chosen": 0.10914154526609812,
|
|
"rewards/margins": 0.22083450167471275,
|
|
"rewards/rejected": -0.11169295640861464,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 11.686853408813477,
|
|
"kl": 8.025674819946289,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35972863.21712538,
|
|
"logits/rejected": -36619921.58466454,
|
|
"logps/chosen": -473.3296062691131,
|
|
"logps/rejected": -364.81689297124603,
|
|
"loss": 0.513,
|
|
"loss/base_kto": 3.7637269496917725,
|
|
"loss/total_with_kl": 4.10407018661499,
|
|
"metrics/advantage_var": 204.41030883789062,
|
|
"regularization/advantage_var": 204.41030883789062,
|
|
"regularization/kl": 0.3403431475162506,
|
|
"rewards/chosen": 0.12917848172902333,
|
|
"rewards/margins": 0.2148514818331536,
|
|
"rewards/rejected": -0.08567300010413027,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 14.738349914550781,
|
|
"kl": 22.37525177001953,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36838246.08588957,
|
|
"logits/rejected": -37229610.39490446,
|
|
"logps/chosen": -459.6443155674847,
|
|
"logps/rejected": -362.95133359872614,
|
|
"loss": 0.5194,
|
|
"loss/base_kto": 3.739776611328125,
|
|
"loss/total_with_kl": 4.155327796936035,
|
|
"metrics/advantage_var": 249.580322265625,
|
|
"regularization/advantage_var": 249.580322265625,
|
|
"regularization/kl": 0.41555124521255493,
|
|
"rewards/chosen": 0.49420952358128833,
|
|
"rewards/margins": 0.2341804900904817,
|
|
"rewards/rejected": 0.26002903349080664,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 21.171537399291992,
|
|
"kl": 6.442798137664795,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -38153568.88615385,
|
|
"logits/rejected": -37634811.93650793,
|
|
"logps/chosen": -492.46552884615386,
|
|
"logps/rejected": -402.9547123015873,
|
|
"loss": 0.5232,
|
|
"loss/base_kto": 3.768808126449585,
|
|
"loss/total_with_kl": 4.185763835906982,
|
|
"metrics/advantage_var": 250.42373657226562,
|
|
"regularization/advantage_var": 250.42373657226562,
|
|
"regularization/kl": 0.41695547103881836,
|
|
"rewards/chosen": 0.12124201847956731,
|
|
"rewards/margins": 0.22917876861907624,
|
|
"rewards/rejected": -0.10793675013950893,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.35466957092285,
|
|
"kl": 14.847846031188965,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37426357.65149137,
|
|
"logits/rejected": -37554720.646967344,
|
|
"logps/chosen": -491.2154140502355,
|
|
"logps/rejected": -377.91959078538105,
|
|
"loss": 0.513,
|
|
"loss/base_kto": 3.7048721313476562,
|
|
"loss/total_with_kl": 4.104198932647705,
|
|
"metrics/advantage_var": 239.83572387695312,
|
|
"regularization/advantage_var": 239.83572387695312,
|
|
"regularization/kl": 0.39932647347450256,
|
|
"rewards/chosen": 0.3274388335750466,
|
|
"rewards/margins": 0.27253533746610903,
|
|
"rewards/rejected": 0.0549034961089376,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 14.543070793151855,
|
|
"kl": 9.724944114685059,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -38061964.01294498,
|
|
"logits/rejected": -37995721.08761329,
|
|
"logps/chosen": -479.9089300161812,
|
|
"logps/rejected": -380.9813302492447,
|
|
"loss": 0.5211,
|
|
"loss/base_kto": 3.771014928817749,
|
|
"loss/total_with_kl": 4.168400287628174,
|
|
"metrics/advantage_var": 238.66995239257812,
|
|
"regularization/advantage_var": 238.66995239257812,
|
|
"regularization/kl": 0.39738544821739197,
|
|
"rewards/chosen": 0.08485675552516307,
|
|
"rewards/margins": 0.20585684375876506,
|
|
"rewards/rejected": -0.121000088233602,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 13.574305534362793,
|
|
"kl": 4.031430244445801,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36701801.06840391,
|
|
"logits/rejected": -38264306.16216216,
|
|
"logps/chosen": -517.4893118892509,
|
|
"logps/rejected": -373.8319725975976,
|
|
"loss": 0.5312,
|
|
"loss/base_kto": 3.6971702575683594,
|
|
"loss/total_with_kl": 4.249903678894043,
|
|
"metrics/advantage_var": 331.9718933105469,
|
|
"regularization/advantage_var": 331.9718933105469,
|
|
"regularization/kl": 0.5527331829071045,
|
|
"rewards/chosen": 0.005228100071512527,
|
|
"rewards/margins": 0.3163857848559063,
|
|
"rewards/rejected": -0.31115768478439376,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 13.513827323913574,
|
|
"kl": 4.463442325592041,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37260558.336,
|
|
"logits/rejected": -38528351.75572519,
|
|
"logps/chosen": -516.85235,
|
|
"logps/rejected": -391.84179389312976,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.688295841217041,
|
|
"loss/total_with_kl": 4.121883869171143,
|
|
"metrics/advantage_var": 260.41314697265625,
|
|
"regularization/advantage_var": 260.41314697265625,
|
|
"regularization/kl": 0.43358784914016724,
|
|
"rewards/chosen": -0.0251224365234375,
|
|
"rewards/margins": 0.29139130057997376,
|
|
"rewards/rejected": -0.31651373710341124,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 24.14131736755371,
|
|
"kl": 9.625970840454102,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36249331.2,
|
|
"logits/rejected": -37038972.8,
|
|
"logps/chosen": -496.13232421875,
|
|
"logps/rejected": -351.006103515625,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.6908395290374756,
|
|
"loss/total_with_kl": 4.157100677490234,
|
|
"metrics/advantage_var": 280.03692626953125,
|
|
"regularization/advantage_var": 280.03692626953125,
|
|
"regularization/kl": 0.46626147627830505,
|
|
"rewards/chosen": 0.17061314582824708,
|
|
"rewards/margins": 0.2945253849029541,
|
|
"rewards/rejected": -0.12391223907470703,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 11.129487037658691,
|
|
"kl": 9.196514129638672,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35751716.79510704,
|
|
"logits/rejected": -38377930.01916933,
|
|
"logps/chosen": -488.51548165137615,
|
|
"logps/rejected": -384.44391473642173,
|
|
"loss": 0.5155,
|
|
"loss/base_kto": 3.716064214706421,
|
|
"loss/total_with_kl": 4.124101638793945,
|
|
"metrics/advantage_var": 245.0675811767578,
|
|
"regularization/advantage_var": 245.0675811767578,
|
|
"regularization/kl": 0.40803757309913635,
|
|
"rewards/chosen": 0.1832698390389072,
|
|
"rewards/margins": 0.2757670644255542,
|
|
"rewards/rejected": -0.092497225386647,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 22.43018341064453,
|
|
"kl": 12.250201225280762,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -38356781.868300155,
|
|
"logits/rejected": -38495018.05422647,
|
|
"logps/chosen": -492.09829632465545,
|
|
"logps/rejected": -379.5605063795853,
|
|
"loss": 0.5268,
|
|
"loss/base_kto": 3.7636144161224365,
|
|
"loss/total_with_kl": 4.214238166809082,
|
|
"metrics/advantage_var": 270.644775390625,
|
|
"regularization/advantage_var": 270.644775390625,
|
|
"regularization/kl": 0.4506235122680664,
|
|
"rewards/chosen": 0.1772036588940467,
|
|
"rewards/margins": 0.2204080561898875,
|
|
"rewards/rejected": -0.04320439729584081,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 13.243584632873535,
|
|
"kl": 5.757604122161865,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36348094.35897436,
|
|
"logits/rejected": -37313329.951219514,
|
|
"logps/chosen": -496.2188501602564,
|
|
"logps/rejected": -385.8954363567073,
|
|
"loss": 0.5144,
|
|
"loss/base_kto": 3.688467025756836,
|
|
"loss/total_with_kl": 4.115045070648193,
|
|
"metrics/advantage_var": 256.2031555175781,
|
|
"regularization/advantage_var": 256.2031555175781,
|
|
"regularization/kl": 0.42657825350761414,
|
|
"rewards/chosen": 0.04293676523061899,
|
|
"rewards/margins": 0.2976913666859353,
|
|
"rewards/rejected": -0.2547546014553163,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 12.686750411987305,
|
|
"kl": 15.58062744140625,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35558159.52721617,
|
|
"logits/rejected": -36753260.10675039,
|
|
"logps/chosen": -482.5460244945568,
|
|
"logps/rejected": -352.4579326923077,
|
|
"loss": 0.507,
|
|
"loss/base_kto": 3.646033525466919,
|
|
"loss/total_with_kl": 4.056111812591553,
|
|
"metrics/advantage_var": 246.2930145263672,
|
|
"regularization/advantage_var": 246.2930145263672,
|
|
"regularization/kl": 0.41007786989212036,
|
|
"rewards/chosen": 0.30974041358864696,
|
|
"rewards/margins": 0.3186565228743459,
|
|
"rewards/rejected": -0.008916109285698954,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 17.221704483032227,
|
|
"kl": 8.498274803161621,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35504890.48615385,
|
|
"logits/rejected": -37986167.46666667,
|
|
"logps/chosen": -507.3598076923077,
|
|
"logps/rejected": -365.4057043650794,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.6991288661956787,
|
|
"loss/total_with_kl": 4.143229961395264,
|
|
"metrics/advantage_var": 266.7270812988281,
|
|
"regularization/advantage_var": 266.7270812988281,
|
|
"regularization/kl": 0.44410058856010437,
|
|
"rewards/chosen": 0.21737396240234375,
|
|
"rewards/margins": 0.2991660914345393,
|
|
"rewards/rejected": -0.08179212903219556,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 13.718184471130371,
|
|
"kl": 14.491968154907227,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36749078.12345679,
|
|
"logits/rejected": -37138393.11392405,
|
|
"logps/chosen": -509.62046682098764,
|
|
"logps/rejected": -389.13963607594934,
|
|
"loss": 0.5166,
|
|
"loss/base_kto": 3.6694633960723877,
|
|
"loss/total_with_kl": 4.133078098297119,
|
|
"metrics/advantage_var": 278.4474182128906,
|
|
"regularization/advantage_var": 278.4474182128906,
|
|
"regularization/kl": 0.46361494064331055,
|
|
"rewards/chosen": 0.31231743612407165,
|
|
"rewards/margins": 0.2795763572765004,
|
|
"rewards/rejected": 0.03274107884757126,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 26.9871883392334,
|
|
"kl": 7.3279314041137695,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36915111.949044585,
|
|
"logits/rejected": -37835179.19018405,
|
|
"logps/chosen": -489.000398089172,
|
|
"logps/rejected": -378.2655051763804,
|
|
"loss": 0.522,
|
|
"loss/base_kto": 3.734973907470703,
|
|
"loss/total_with_kl": 4.17586612701416,
|
|
"metrics/advantage_var": 264.79986572265625,
|
|
"regularization/advantage_var": 264.79986572265625,
|
|
"regularization/kl": 0.44089174270629883,
|
|
"rewards/chosen": 0.06606191890254902,
|
|
"rewards/margins": 0.24931300143406265,
|
|
"rewards/rejected": -0.18325108253151362,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 22.35744285583496,
|
|
"kl": 8.32131576538086,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36016670.30529595,
|
|
"logits/rejected": -36874018.50783699,
|
|
"logps/chosen": -475.8301207165109,
|
|
"logps/rejected": -366.3892780172414,
|
|
"loss": 0.5086,
|
|
"loss/base_kto": 3.609822988510132,
|
|
"loss/total_with_kl": 4.068904399871826,
|
|
"metrics/advantage_var": 275.7244873046875,
|
|
"regularization/advantage_var": 275.7244873046875,
|
|
"regularization/kl": 0.4590812623500824,
|
|
"rewards/chosen": 0.1779957946587203,
|
|
"rewards/margins": 0.39229108924997225,
|
|
"rewards/rejected": -0.21429529459125196,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.741931915283203,
|
|
"kl": 7.286538124084473,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -35493546.666666664,
|
|
"logits/rejected": -36181456.298136644,
|
|
"logps/chosen": -473.7684748427673,
|
|
"logps/rejected": -382.61563470496895,
|
|
"loss": 0.5218,
|
|
"loss/base_kto": 3.7576091289520264,
|
|
"loss/total_with_kl": 4.174242973327637,
|
|
"metrics/advantage_var": 250.2304229736328,
|
|
"regularization/advantage_var": 250.2304229736328,
|
|
"regularization/kl": 0.41663360595703125,
|
|
"rewards/chosen": 0.007626329577943814,
|
|
"rewards/margins": 0.22689612354448593,
|
|
"rewards/rejected": -0.21926979396654211,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 12.266626358032227,
|
|
"kl": 18.002002716064453,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35223971.62079511,
|
|
"logits/rejected": -35371835.70607029,
|
|
"logps/chosen": -470.1541475535168,
|
|
"logps/rejected": -370.6930910543131,
|
|
"loss": 0.5074,
|
|
"loss/base_kto": 3.6133172512054443,
|
|
"loss/total_with_kl": 4.058897495269775,
|
|
"metrics/advantage_var": 267.615966796875,
|
|
"regularization/advantage_var": 267.615966796875,
|
|
"regularization/kl": 0.4455806314945221,
|
|
"rewards/chosen": 0.41311981481149657,
|
|
"rewards/margins": 0.35102737804235845,
|
|
"rewards/rejected": 0.06209243676913813,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 17.56349754333496,
|
|
"kl": 11.449131965637207,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35984177.88083736,
|
|
"logits/rejected": -36451907.593323216,
|
|
"logps/chosen": -460.2981078904992,
|
|
"logps/rejected": -367.9440440060698,
|
|
"loss": 0.5126,
|
|
"loss/base_kto": 3.6980767250061035,
|
|
"loss/total_with_kl": 4.1010823249816895,
|
|
"metrics/advantage_var": 242.0452117919922,
|
|
"regularization/advantage_var": 242.0452117919922,
|
|
"regularization/kl": 0.40300533175468445,
|
|
"rewards/chosen": 0.17118457050907054,
|
|
"rewards/margins": 0.27623399324060033,
|
|
"rewards/rejected": -0.10504942273152978,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 19.45347023010254,
|
|
"kl": 12.99169635772705,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36561224.653789006,
|
|
"logits/rejected": -38311716.691927515,
|
|
"logps/chosen": -501.74219910846955,
|
|
"logps/rejected": -379.17285317133445,
|
|
"loss": 0.517,
|
|
"loss/base_kto": 3.6527276039123535,
|
|
"loss/total_with_kl": 4.13591194152832,
|
|
"metrics/advantage_var": 290.2008361816406,
|
|
"regularization/advantage_var": 290.2008361816406,
|
|
"regularization/kl": 0.4831843972206116,
|
|
"rewards/chosen": 0.2943211367013373,
|
|
"rewards/margins": 0.33720744022122573,
|
|
"rewards/rejected": -0.04288630351988841,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.827146530151367,
|
|
"kl": 16.641677856445312,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36507987.2,
|
|
"logits/rejected": -38247452.8,
|
|
"logps/chosen": -485.74462890625,
|
|
"logps/rejected": -382.705224609375,
|
|
"loss": 0.5162,
|
|
"loss/base_kto": 3.6911838054656982,
|
|
"loss/total_with_kl": 4.129607677459717,
|
|
"metrics/advantage_var": 263.3175354003906,
|
|
"regularization/advantage_var": 263.3175354003906,
|
|
"regularization/kl": 0.43842363357543945,
|
|
"rewards/chosen": 0.30822157859802246,
|
|
"rewards/margins": 0.25912770330905915,
|
|
"rewards/rejected": 0.04909387528896332,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.580856323242188,
|
|
"kl": 10.5079984664917,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36100899.88785047,
|
|
"logits/rejected": -37838867.26018809,
|
|
"logps/chosen": -472.3942757009346,
|
|
"logps/rejected": -355.6166242163009,
|
|
"loss": 0.5186,
|
|
"loss/base_kto": 3.6576321125030518,
|
|
"loss/total_with_kl": 4.148627281188965,
|
|
"metrics/advantage_var": 294.8919982910156,
|
|
"regularization/advantage_var": 294.8919982910156,
|
|
"regularization/kl": 0.4909951388835907,
|
|
"rewards/chosen": 0.22137812439154986,
|
|
"rewards/margins": 0.3284851493467015,
|
|
"rewards/rejected": -0.10710702495515159,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 14.14950942993164,
|
|
"kl": 17.051233291625977,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36647768.26209048,
|
|
"logits/rejected": -38771062.98591549,
|
|
"logps/chosen": -490.13855304212166,
|
|
"logps/rejected": -355.97674589201876,
|
|
"loss": 0.5155,
|
|
"loss/base_kto": 3.6732146739959717,
|
|
"loss/total_with_kl": 4.1241068840026855,
|
|
"metrics/advantage_var": 270.8059997558594,
|
|
"regularization/advantage_var": 270.8059997558594,
|
|
"regularization/kl": 0.45089197158813477,
|
|
"rewards/chosen": 0.32415773864841313,
|
|
"rewards/margins": 0.2774173992528274,
|
|
"rewards/rejected": 0.04674033939558576,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 14.010693550109863,
|
|
"kl": 13.791582107543945,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37254645.71251932,
|
|
"logits/rejected": -37721688.164297,
|
|
"logps/chosen": -492.6201700154559,
|
|
"logps/rejected": -363.3092910742496,
|
|
"loss": 0.5102,
|
|
"loss/base_kto": 3.697603702545166,
|
|
"loss/total_with_kl": 4.081522464752197,
|
|
"metrics/advantage_var": 230.5818328857422,
|
|
"regularization/advantage_var": 230.5818328857422,
|
|
"regularization/kl": 0.38391873240470886,
|
|
"rewards/chosen": 0.24241433003586263,
|
|
"rewards/margins": 0.27103954713010325,
|
|
"rewards/rejected": -0.028625217094240596,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 19.1981143951416,
|
|
"kl": 16.53029441833496,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35613101.51975684,
|
|
"logits/rejected": -37462394.649517685,
|
|
"logps/chosen": -465.6258548632219,
|
|
"logps/rejected": -376.64986937299034,
|
|
"loss": 0.52,
|
|
"loss/base_kto": 3.647289991378784,
|
|
"loss/total_with_kl": 4.16015100479126,
|
|
"metrics/advantage_var": 308.0245056152344,
|
|
"regularization/advantage_var": 308.0245056152344,
|
|
"regularization/kl": 0.5128607749938965,
|
|
"rewards/chosen": 0.3199482138033696,
|
|
"rewards/margins": 0.3004506781836645,
|
|
"rewards/rejected": 0.019497535619705054,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 18.38774871826172,
|
|
"kl": 10.443411827087402,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36128941.55932204,
|
|
"logits/rejected": -37185438.63074485,
|
|
"logps/chosen": -482.4111132511556,
|
|
"logps/rejected": -384.6418878763867,
|
|
"loss": 0.5168,
|
|
"loss/base_kto": 3.6472573280334473,
|
|
"loss/total_with_kl": 4.134653568267822,
|
|
"metrics/advantage_var": 292.73046875,
|
|
"regularization/advantage_var": 292.73046875,
|
|
"regularization/kl": 0.487396240234375,
|
|
"rewards/chosen": 0.22631828884130634,
|
|
"rewards/margins": 0.3277210733309093,
|
|
"rewards/rejected": -0.10140278448960294,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 15.38276481628418,
|
|
"kl": 20.686330795288086,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -34609602.43037975,
|
|
"logits/rejected": -35595169.18518519,
|
|
"logps/chosen": -470.78317840189874,
|
|
"logps/rejected": -368.62647087191357,
|
|
"loss": 0.5163,
|
|
"loss/base_kto": 3.6479694843292236,
|
|
"loss/total_with_kl": 4.130124092102051,
|
|
"metrics/advantage_var": 289.5821838378906,
|
|
"regularization/advantage_var": 289.5821838378906,
|
|
"regularization/kl": 0.48215436935424805,
|
|
"rewards/chosen": 0.376678466796875,
|
|
"rewards/margins": 0.2798532909817166,
|
|
"rewards/rejected": 0.09682517581515843,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 17.76531410217285,
|
|
"kl": 22.319847106933594,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35493900.84012539,
|
|
"logits/rejected": -36400628.83489097,
|
|
"logps/chosen": -464.8445336990596,
|
|
"logps/rejected": -353.57797897196264,
|
|
"loss": 0.5119,
|
|
"loss/base_kto": 3.6981751918792725,
|
|
"loss/total_with_kl": 4.095587253570557,
|
|
"metrics/advantage_var": 238.68589782714844,
|
|
"regularization/advantage_var": 238.68589782714844,
|
|
"regularization/kl": 0.3974120020866394,
|
|
"rewards/chosen": 0.26839138497379506,
|
|
"rewards/margins": 0.2261361170551281,
|
|
"rewards/rejected": 0.04225526791866695,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 13.99153995513916,
|
|
"kl": 8.310779571533203,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36944271.92440945,
|
|
"logits/rejected": -37181252.66356589,
|
|
"logps/chosen": -493.5645177165354,
|
|
"logps/rejected": -368.17020348837207,
|
|
"loss": 0.5202,
|
|
"loss/base_kto": 3.6871566772460938,
|
|
"loss/total_with_kl": 4.16163969039917,
|
|
"metrics/advantage_var": 284.97454833984375,
|
|
"regularization/advantage_var": 284.97454833984375,
|
|
"regularization/kl": 0.47448259592056274,
|
|
"rewards/chosen": 0.14224445012610729,
|
|
"rewards/margins": 0.2977979159767008,
|
|
"rewards/rejected": -0.15555346585059351,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 21.295419692993164,
|
|
"kl": 18.66389274597168,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35438458.22706065,
|
|
"logits/rejected": -37468340.04395604,
|
|
"logps/chosen": -474.6709758942457,
|
|
"logps/rejected": -387.30349784144425,
|
|
"loss": 0.5107,
|
|
"loss/base_kto": 3.630136489868164,
|
|
"loss/total_with_kl": 4.085545539855957,
|
|
"metrics/advantage_var": 273.518798828125,
|
|
"regularization/advantage_var": 273.518798828125,
|
|
"regularization/kl": 0.45540881156921387,
|
|
"rewards/chosen": 0.3340433455922555,
|
|
"rewards/margins": 0.3193941086227344,
|
|
"rewards/rejected": 0.014649236969521131,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 13.512069702148438,
|
|
"kl": 19.664888381958008,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -34999406.8368336,
|
|
"logits/rejected": -35503877.03479576,
|
|
"logps/chosen": -474.47294022617126,
|
|
"logps/rejected": -359.9348052193646,
|
|
"loss": 0.51,
|
|
"loss/base_kto": 3.583148241043091,
|
|
"loss/total_with_kl": 4.080239295959473,
|
|
"metrics/advantage_var": 298.5530700683594,
|
|
"regularization/advantage_var": 298.5530700683594,
|
|
"regularization/kl": 0.49709081649780273,
|
|
"rewards/chosen": 0.41158145540943053,
|
|
"rewards/margins": 0.35769089818523186,
|
|
"rewards/rejected": 0.05389055722419866,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 13.908814430236816,
|
|
"kl": 11.910075187683105,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36904684.612055644,
|
|
"logits/rejected": -37676273.80031696,
|
|
"logps/chosen": -505.07293276661517,
|
|
"logps/rejected": -375.7888272583201,
|
|
"loss": 0.5197,
|
|
"loss/base_kto": 3.493528127670288,
|
|
"loss/total_with_kl": 4.15781831741333,
|
|
"metrics/advantage_var": 398.9729919433594,
|
|
"regularization/advantage_var": 398.9729919433594,
|
|
"regularization/kl": 0.6642900109291077,
|
|
"rewards/chosen": 0.2728085879011906,
|
|
"rewards/margins": 0.5144740970601397,
|
|
"rewards/rejected": -0.24166550915894908,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 14.702442169189453,
|
|
"kl": 14.928006172180176,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35880108.8,
|
|
"logits/rejected": -37205433.6,
|
|
"logps/chosen": -475.6765625,
|
|
"logps/rejected": -373.49873046875,
|
|
"loss": 0.5123,
|
|
"loss/base_kto": 3.121844530105591,
|
|
"loss/total_with_kl": 4.098536014556885,
|
|
"metrics/advantage_var": 586.6016235351562,
|
|
"regularization/advantage_var": 586.6016235351562,
|
|
"regularization/kl": 0.9766916632652283,
|
|
"rewards/chosen": 0.5970371723175049,
|
|
"rewards/margins": 0.9430038928985596,
|
|
"rewards/rejected": -0.3459667205810547,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 16.506349563598633,
|
|
"kl": 23.031673431396484,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35266343.94056463,
|
|
"logits/rejected": -36923267.1630972,
|
|
"logps/chosen": -482.32345839524515,
|
|
"logps/rejected": -372.7668863261944,
|
|
"loss": 0.532,
|
|
"loss/base_kto": 3.128161668777466,
|
|
"loss/total_with_kl": 4.256258487701416,
|
|
"metrics/advantage_var": 677.5355834960938,
|
|
"regularization/advantage_var": 677.5355834960938,
|
|
"regularization/kl": 1.1280968189239502,
|
|
"rewards/chosen": 0.7322181826360513,
|
|
"rewards/margins": 0.951558021559548,
|
|
"rewards/rejected": -0.2193398389234967,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 23.76866912841797,
|
|
"kl": 16.244009017944336,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36483396.05063291,
|
|
"logits/rejected": -36804020.14814815,
|
|
"logps/chosen": -473.2432753164557,
|
|
"logps/rejected": -365.21489197530866,
|
|
"loss": 0.5105,
|
|
"loss/base_kto": 3.1461238861083984,
|
|
"loss/total_with_kl": 4.083627223968506,
|
|
"metrics/advantage_var": 563.065185546875,
|
|
"regularization/advantage_var": 563.065185546875,
|
|
"regularization/kl": 0.9375035166740417,
|
|
"rewards/chosen": 0.587558553188662,
|
|
"rewards/margins": 0.9262237736612098,
|
|
"rewards/rejected": -0.33866522047254777,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 15.511030197143555,
|
|
"kl": 13.907042503356934,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -34902893.71428572,
|
|
"logits/rejected": -37057144.180064306,
|
|
"logps/chosen": -488.1994680851064,
|
|
"logps/rejected": -373.65966639871385,
|
|
"loss": 0.5429,
|
|
"loss/base_kto": 3.1668577194213867,
|
|
"loss/total_with_kl": 4.342853546142578,
|
|
"metrics/advantage_var": 706.3037109375,
|
|
"regularization/advantage_var": 706.3037109375,
|
|
"regularization/kl": 1.1759957075119019,
|
|
"rewards/chosen": 0.612727527560434,
|
|
"rewards/margins": 0.9324909564723362,
|
|
"rewards/rejected": -0.3197634289119021,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 22.089319229125977,
|
|
"kl": 14.627352714538574,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -33950064.64,
|
|
"logits/rejected": -36507017.34603175,
|
|
"logps/chosen": -490.36788461538464,
|
|
"logps/rejected": -360.1581349206349,
|
|
"loss": 0.5173,
|
|
"loss/base_kto": 3.166754961013794,
|
|
"loss/total_with_kl": 4.138631343841553,
|
|
"metrics/advantage_var": 583.7092895507812,
|
|
"regularization/advantage_var": 583.7092895507812,
|
|
"regularization/kl": 0.9718759655952454,
|
|
"rewards/chosen": 0.583619384765625,
|
|
"rewards/margins": 0.8808749583410839,
|
|
"rewards/rejected": -0.29725557357545884,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 11.542262077331543,
|
|
"kl": 24.28908348083496,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34886517.36615384,
|
|
"logits/rejected": -36736503.87301587,
|
|
"logps/chosen": -486.0973076923077,
|
|
"logps/rejected": -356.49652777777777,
|
|
"loss": 0.4974,
|
|
"loss/base_kto": 3.090975046157837,
|
|
"loss/total_with_kl": 3.9790139198303223,
|
|
"metrics/advantage_var": 533.3565673828125,
|
|
"regularization/advantage_var": 533.3565673828125,
|
|
"regularization/kl": 0.8880386352539062,
|
|
"rewards/chosen": 0.7821881103515625,
|
|
"rewards/margins": 0.930651344420418,
|
|
"rewards/rejected": -0.1484632340688554,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 20.099254608154297,
|
|
"kl": 24.87495994567871,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35106203.26504065,
|
|
"logits/rejected": -37627171.03157895,
|
|
"logps/chosen": -472.17850609756096,
|
|
"logps/rejected": -393.18529135338343,
|
|
"loss": 0.4981,
|
|
"loss/base_kto": 3.1312081813812256,
|
|
"loss/total_with_kl": 3.9850945472717285,
|
|
"metrics/advantage_var": 512.8447265625,
|
|
"regularization/advantage_var": 512.8447265625,
|
|
"regularization/kl": 0.8538865447044373,
|
|
"rewards/chosen": 0.6398583885130843,
|
|
"rewards/margins": 0.8641956293201801,
|
|
"rewards/rejected": -0.22433724080709586,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.070850372314453,
|
|
"kl": 8.647613525390625,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34613633.064891845,
|
|
"logits/rejected": -35375137.17820324,
|
|
"logps/chosen": -492.28670965058234,
|
|
"logps/rejected": -387.3687868188513,
|
|
"loss": 0.5141,
|
|
"loss/base_kto": 3.2170650959014893,
|
|
"loss/total_with_kl": 4.112492561340332,
|
|
"metrics/advantage_var": 537.7942504882812,
|
|
"regularization/advantage_var": 537.7942504882812,
|
|
"regularization/kl": 0.895427405834198,
|
|
"rewards/chosen": 0.3722284098036476,
|
|
"rewards/margins": 0.8449529310866695,
|
|
"rewards/rejected": -0.4727245212830219,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 10.273600578308105,
|
|
"kl": 7.31499719619751,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35378362.7672956,
|
|
"logits/rejected": -37036019.27950311,
|
|
"logps/chosen": -495.50348860062894,
|
|
"logps/rejected": -370.22049689440996,
|
|
"loss": 0.5211,
|
|
"loss/base_kto": 3.146202802658081,
|
|
"loss/total_with_kl": 4.16919469833374,
|
|
"metrics/advantage_var": 614.4094848632812,
|
|
"regularization/advantage_var": 614.4094848632812,
|
|
"regularization/kl": 1.0229917764663696,
|
|
"rewards/chosen": 0.39650937446258355,
|
|
"rewards/margins": 0.944580151739612,
|
|
"rewards/rejected": -0.5480707772770284,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 20.26920509338379,
|
|
"kl": 8.604382514953613,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34521154.49350649,
|
|
"logits/rejected": -36876016.578313254,
|
|
"logps/chosen": -501.86079545454544,
|
|
"logps/rejected": -404.08621987951807,
|
|
"loss": 0.5214,
|
|
"loss/base_kto": 3.1481845378875732,
|
|
"loss/total_with_kl": 4.171576976776123,
|
|
"metrics/advantage_var": 614.6500244140625,
|
|
"regularization/advantage_var": 614.6500244140625,
|
|
"regularization/kl": 1.0233923196792603,
|
|
"rewards/chosen": 0.42700284487241275,
|
|
"rewards/margins": 0.9547071067441184,
|
|
"rewards/rejected": -0.5277042618717056,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 37.875823974609375,
|
|
"kl": 9.719233512878418,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34423242.60122699,
|
|
"logits/rejected": -35191530.80254777,
|
|
"logps/chosen": -481.0804256134969,
|
|
"logps/rejected": -384.8274034633758,
|
|
"loss": 0.5025,
|
|
"loss/base_kto": 3.176201581954956,
|
|
"loss/total_with_kl": 4.0201544761657715,
|
|
"metrics/advantage_var": 506.8787536621094,
|
|
"regularization/advantage_var": 506.8787536621094,
|
|
"regularization/kl": 0.8439531326293945,
|
|
"rewards/chosen": 0.4389552953053106,
|
|
"rewards/margins": 0.8862632950316243,
|
|
"rewards/rejected": -0.4473079997263137,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 11.402820587158203,
|
|
"kl": 18.130916595458984,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35122157.38181818,
|
|
"logits/rejected": -37554199.12258065,
|
|
"logps/chosen": -490.7692234848485,
|
|
"logps/rejected": -362.0521673387097,
|
|
"loss": 0.5018,
|
|
"loss/base_kto": 3.152398109436035,
|
|
"loss/total_with_kl": 4.01436185836792,
|
|
"metrics/advantage_var": 517.6959228515625,
|
|
"regularization/advantage_var": 517.6959228515625,
|
|
"regularization/kl": 0.86196368932724,
|
|
"rewards/chosen": 0.658410459576231,
|
|
"rewards/margins": 0.8959365471833378,
|
|
"rewards/rejected": -0.23752608760710686,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 29.655818939208984,
|
|
"kl": 15.274200439453125,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35440509.817054264,
|
|
"logits/rejected": -37492610.217322834,
|
|
"logps/chosen": -489.56337209302325,
|
|
"logps/rejected": -368.6337598425197,
|
|
"loss": 0.5293,
|
|
"loss/base_kto": 3.1672253608703613,
|
|
"loss/total_with_kl": 4.2346062660217285,
|
|
"metrics/advantage_var": 641.0697631835938,
|
|
"regularization/advantage_var": 641.0697631835938,
|
|
"regularization/kl": 1.0673811435699463,
|
|
"rewards/chosen": 0.6777743080789729,
|
|
"rewards/margins": 0.8876981438934414,
|
|
"rewards/rejected": -0.2099238358144685,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 16.591093063354492,
|
|
"kl": 12.736869812011719,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35436795.96214511,
|
|
"logits/rejected": -37583494.7368421,
|
|
"logps/chosen": -489.9379929022082,
|
|
"logps/rejected": -353.62091234520125,
|
|
"loss": 0.5083,
|
|
"loss/base_kto": 3.1960511207580566,
|
|
"loss/total_with_kl": 4.066004276275635,
|
|
"metrics/advantage_var": 522.4944458007812,
|
|
"regularization/advantage_var": 522.4944458007812,
|
|
"regularization/kl": 0.8699531555175781,
|
|
"rewards/chosen": 0.5690563213938042,
|
|
"rewards/margins": 0.8557566852615964,
|
|
"rewards/rejected": -0.2867003638677922,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 14.795488357543945,
|
|
"kl": 14.876036643981934,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34242087.384615384,
|
|
"logits/rejected": -35085061.972006224,
|
|
"logps/chosen": -470.746762166405,
|
|
"logps/rejected": -392.40875291601867,
|
|
"loss": 0.5059,
|
|
"loss/base_kto": 3.1795268058776855,
|
|
"loss/total_with_kl": 4.047567844390869,
|
|
"metrics/advantage_var": 521.3460083007812,
|
|
"regularization/advantage_var": 521.3460083007812,
|
|
"regularization/kl": 0.8680410385131836,
|
|
"rewards/chosen": 0.4943622008223362,
|
|
"rewards/margins": 0.8517980531578848,
|
|
"rewards/rejected": -0.3574358523355487,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 21.76078224182129,
|
|
"kl": 14.435269355773926,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36055591.503782146,
|
|
"logits/rejected": -36014606.061389334,
|
|
"logps/chosen": -467.8831316187595,
|
|
"logps/rejected": -364.53715670436185,
|
|
"loss": 0.5036,
|
|
"loss/base_kto": 3.1299355030059814,
|
|
"loss/total_with_kl": 4.029140472412109,
|
|
"metrics/advantage_var": 540.0630493164062,
|
|
"regularization/advantage_var": 540.0630493164062,
|
|
"regularization/kl": 0.8992050290107727,
|
|
"rewards/chosen": 0.664907296739079,
|
|
"rewards/margins": 0.9567524427580486,
|
|
"rewards/rejected": -0.2918451460189696,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 13.759382247924805,
|
|
"kl": 12.645831108093262,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35678454.668779716,
|
|
"logits/rejected": -37359751.69183359,
|
|
"logps/chosen": -477.5898375594295,
|
|
"logps/rejected": -400.84473709553157,
|
|
"loss": 0.5141,
|
|
"loss/base_kto": 3.1888949871063232,
|
|
"loss/total_with_kl": 4.1127119064331055,
|
|
"metrics/advantage_var": 554.844970703125,
|
|
"regularization/advantage_var": 554.844970703125,
|
|
"regularization/kl": 0.9238168597221375,
|
|
"rewards/chosen": 0.5091591425440769,
|
|
"rewards/margins": 0.8849552328332237,
|
|
"rewards/rejected": -0.3757960902891468,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 11.001599311828613,
|
|
"kl": 23.096729278564453,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -36102240.90851735,
|
|
"logits/rejected": -35913325.374613,
|
|
"logps/chosen": -472.0725059148265,
|
|
"logps/rejected": -378.8892221362229,
|
|
"loss": 0.5082,
|
|
"loss/base_kto": 3.215791702270508,
|
|
"loss/total_with_kl": 4.065337657928467,
|
|
"metrics/advantage_var": 510.238037109375,
|
|
"regularization/advantage_var": 510.238037109375,
|
|
"regularization/kl": 0.8495463728904724,
|
|
"rewards/chosen": 0.6146769719168967,
|
|
"rewards/margins": 0.817916436455282,
|
|
"rewards/rejected": -0.20323946453838526,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 10.352461814880371,
|
|
"kl": 11.740008354187012,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34013443.9628483,
|
|
"logits/rejected": -36186454.41009463,
|
|
"logps/chosen": -504.7594814241486,
|
|
"logps/rejected": -361.39126577287067,
|
|
"loss": 0.5183,
|
|
"loss/base_kto": 3.1847331523895264,
|
|
"loss/total_with_kl": 4.146283149719238,
|
|
"metrics/advantage_var": 577.5076904296875,
|
|
"regularization/advantage_var": 577.5076904296875,
|
|
"regularization/kl": 0.9615503549575806,
|
|
"rewards/chosen": 0.5925627433847717,
|
|
"rewards/margins": 0.9020782782760868,
|
|
"rewards/rejected": -0.30951553489131506,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 13.596035957336426,
|
|
"kl": 16.4085693359375,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35841516.051948056,
|
|
"logits/rejected": -36332016.578313254,
|
|
"logps/chosen": -489.4002637987013,
|
|
"logps/rejected": -392.4166039156627,
|
|
"loss": 0.5065,
|
|
"loss/base_kto": 3.18613338470459,
|
|
"loss/total_with_kl": 4.052160739898682,
|
|
"metrics/advantage_var": 520.1364135742188,
|
|
"regularization/advantage_var": 520.1364135742188,
|
|
"regularization/kl": 0.8660271763801575,
|
|
"rewards/chosen": 0.6012120432667918,
|
|
"rewards/margins": 0.863107280465587,
|
|
"rewards/rejected": -0.2618952371987952,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 27.7114315032959,
|
|
"kl": 9.63242244720459,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33595982.38436482,
|
|
"logits/rejected": -35467869.78978979,
|
|
"logps/chosen": -497.65838762214986,
|
|
"logps/rejected": -363.7683464714715,
|
|
"loss": 0.5203,
|
|
"loss/base_kto": 3.22320556640625,
|
|
"loss/total_with_kl": 4.162144660949707,
|
|
"metrics/advantage_var": 563.9271240234375,
|
|
"regularization/advantage_var": 563.9271240234375,
|
|
"regularization/kl": 0.9389387369155884,
|
|
"rewards/chosen": 0.4438537697062042,
|
|
"rewards/margins": 0.8417984520594559,
|
|
"rewards/rejected": -0.3979446823532517,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 11.680557250976562,
|
|
"kl": 12.713912963867188,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35893399.10832025,
|
|
"logits/rejected": -36816042.40124417,
|
|
"logps/chosen": -470.94485871271587,
|
|
"logps/rejected": -384.1338938569207,
|
|
"loss": 0.5072,
|
|
"loss/base_kto": 3.170074224472046,
|
|
"loss/total_with_kl": 4.057701587677002,
|
|
"metrics/advantage_var": 533.1096801757812,
|
|
"regularization/advantage_var": 533.1096801757812,
|
|
"regularization/kl": 0.8876276016235352,
|
|
"rewards/chosen": 0.49408897986778844,
|
|
"rewards/margins": 0.8758209297882054,
|
|
"rewards/rejected": -0.381731949920417,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 158.68943786621094,
|
|
"kl": 10.79794979095459,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35022391.39672131,
|
|
"logits/rejected": -35362760.979104474,
|
|
"logps/chosen": -451.43954918032784,
|
|
"logps/rejected": -358.0534514925373,
|
|
"loss": 0.5316,
|
|
"loss/base_kto": 3.210341691970825,
|
|
"loss/total_with_kl": 4.25247049331665,
|
|
"metrics/advantage_var": 625.9031372070312,
|
|
"regularization/advantage_var": 625.9031372070312,
|
|
"regularization/kl": 1.0421288013458252,
|
|
"rewards/chosen": 0.4774256972015881,
|
|
"rewards/margins": 0.8543927145319286,
|
|
"rewards/rejected": -0.3769670173303405,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 15.02539348602295,
|
|
"kl": 9.845388412475586,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34552865.623880595,
|
|
"logits/rejected": -35934872.76065574,
|
|
"logps/chosen": -479.00116604477614,
|
|
"logps/rejected": -385.7408299180328,
|
|
"loss": 0.5567,
|
|
"loss/base_kto": 3.2393271923065186,
|
|
"loss/total_with_kl": 4.453500747680664,
|
|
"metrics/advantage_var": 729.2332153320312,
|
|
"regularization/advantage_var": 729.2332153320312,
|
|
"regularization/kl": 1.2141733169555664,
|
|
"rewards/chosen": 0.4126215237290112,
|
|
"rewards/margins": 0.791786522768458,
|
|
"rewards/rejected": -0.3791649990394467,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 12.136699676513672,
|
|
"kl": 14.17646312713623,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34894956.38546603,
|
|
"logits/rejected": -34558613.5641422,
|
|
"logps/chosen": -446.8041074249605,
|
|
"logps/rejected": -353.34309795208657,
|
|
"loss": 0.4965,
|
|
"loss/base_kto": 3.192150831222534,
|
|
"loss/total_with_kl": 3.971637487411499,
|
|
"metrics/advantage_var": 468.1601257324219,
|
|
"regularization/advantage_var": 468.1601257324219,
|
|
"regularization/kl": 0.7794865965843201,
|
|
"rewards/chosen": 0.5511742180557613,
|
|
"rewards/margins": 0.857186897897217,
|
|
"rewards/rejected": -0.30601267984145575,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 17.462207794189453,
|
|
"kl": 9.537064552307129,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33522303.59621451,
|
|
"logits/rejected": -34063895.77708978,
|
|
"logps/chosen": -462.58049093059935,
|
|
"logps/rejected": -351.0721991099071,
|
|
"loss": 0.5011,
|
|
"loss/base_kto": 3.2250454425811768,
|
|
"loss/total_with_kl": 4.008975982666016,
|
|
"metrics/advantage_var": 470.8292541503906,
|
|
"regularization/advantage_var": 470.8292541503906,
|
|
"regularization/kl": 0.7839307188987732,
|
|
"rewards/chosen": 0.44059825620440657,
|
|
"rewards/margins": 0.8339564690799145,
|
|
"rewards/rejected": -0.39335821287550793,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 32.466583251953125,
|
|
"kl": 10.466676712036133,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34457949.01607717,
|
|
"logits/rejected": -35909224.26747721,
|
|
"logps/chosen": -482.7487942122186,
|
|
"logps/rejected": -388.04832351823705,
|
|
"loss": 0.5237,
|
|
"loss/base_kto": 3.182065963745117,
|
|
"loss/total_with_kl": 4.189762115478516,
|
|
"metrics/advantage_var": 605.222900390625,
|
|
"regularization/advantage_var": 605.222900390625,
|
|
"regularization/kl": 1.0076961517333984,
|
|
"rewards/chosen": 0.4184399080429813,
|
|
"rewards/margins": 0.8830769918908157,
|
|
"rewards/rejected": -0.46463708384783436,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 22.69070053100586,
|
|
"kl": 9.186279296875,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35610322.18947368,
|
|
"logits/rejected": -36267322.69268293,
|
|
"logps/chosen": -497.3314379699248,
|
|
"logps/rejected": -380.02962398373984,
|
|
"loss": 0.5444,
|
|
"loss/base_kto": 3.284139394760132,
|
|
"loss/total_with_kl": 4.355058670043945,
|
|
"metrics/advantage_var": 643.19482421875,
|
|
"regularization/advantage_var": 643.19482421875,
|
|
"regularization/kl": 1.070919394493103,
|
|
"rewards/chosen": 0.34557646557800753,
|
|
"rewards/margins": 0.7247198098678957,
|
|
"rewards/rejected": -0.3791433442898882,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 14.113452911376953,
|
|
"kl": 15.024617195129395,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35950959.82952816,
|
|
"logits/rejected": -36128922.50401284,
|
|
"logps/chosen": -506.6966324200913,
|
|
"logps/rejected": -377.0258326645265,
|
|
"loss": 0.5184,
|
|
"loss/base_kto": 3.186739683151245,
|
|
"loss/total_with_kl": 4.147265434265137,
|
|
"metrics/advantage_var": 576.8922119140625,
|
|
"regularization/advantage_var": 576.8922119140625,
|
|
"regularization/kl": 0.9605255126953125,
|
|
"rewards/chosen": 0.6062930497586092,
|
|
"rewards/margins": 0.8413944464937868,
|
|
"rewards/rejected": -0.23510139673517758,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 10.620443344116211,
|
|
"kl": 13.38236141204834,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34308755.2,
|
|
"logits/rejected": -35334502.4,
|
|
"logps/chosen": -470.38818359375,
|
|
"logps/rejected": -373.6876953125,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.1982343196868896,
|
|
"loss/total_with_kl": 4.227118968963623,
|
|
"metrics/advantage_var": 617.9486694335938,
|
|
"regularization/advantage_var": 617.9486694335938,
|
|
"regularization/kl": 1.0288845300674438,
|
|
"rewards/chosen": 0.5007099151611328,
|
|
"rewards/margins": 0.8597929954528809,
|
|
"rewards/rejected": -0.35908308029174807,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 11.011088371276855,
|
|
"kl": 11.875654220581055,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34655813.18918919,
|
|
"logits/rejected": -37005063.47158218,
|
|
"logps/chosen": -465.6476053259141,
|
|
"logps/rejected": -380.7684331797235,
|
|
"loss": 0.5254,
|
|
"loss/base_kto": 3.1463446617126465,
|
|
"loss/total_with_kl": 4.203329086303711,
|
|
"metrics/advantage_var": 634.8256225585938,
|
|
"regularization/advantage_var": 634.8256225585938,
|
|
"regularization/kl": 1.0569846630096436,
|
|
"rewards/chosen": 0.5462430581000348,
|
|
"rewards/margins": 0.9224346265479155,
|
|
"rewards/rejected": -0.37619156844788065,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 30.224512100219727,
|
|
"kl": 11.387033462524414,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36086213.01382489,
|
|
"logits/rejected": -36318222.6518283,
|
|
"logps/chosen": -473.8333813364055,
|
|
"logps/rejected": -388.89139507154215,
|
|
"loss": 0.4966,
|
|
"loss/base_kto": 3.173307180404663,
|
|
"loss/total_with_kl": 3.972987413406372,
|
|
"metrics/advantage_var": 480.2885437011719,
|
|
"regularization/advantage_var": 480.2885437011719,
|
|
"regularization/kl": 0.7996804118156433,
|
|
"rewards/chosen": 0.5204185099096342,
|
|
"rewards/margins": 0.8890296136891951,
|
|
"rewards/rejected": -0.3686111037795608,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 15.557607650756836,
|
|
"kl": 17.34191131591797,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35890980.4544,
|
|
"logits/rejected": -36086357.20305344,
|
|
"logps/chosen": -456.947,
|
|
"logps/rejected": -379.2212786259542,
|
|
"loss": 0.5239,
|
|
"loss/base_kto": 3.226628065109253,
|
|
"loss/total_with_kl": 4.1909074783325195,
|
|
"metrics/advantage_var": 579.1466674804688,
|
|
"regularization/advantage_var": 579.1466674804688,
|
|
"regularization/kl": 0.9642791748046875,
|
|
"rewards/chosen": 0.59911005859375,
|
|
"rewards/margins": 0.8291229318866293,
|
|
"rewards/rejected": -0.23001287329287928,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 12.443046569824219,
|
|
"kl": 13.553735733032227,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34566584.70886076,
|
|
"logits/rejected": -36109356.24691358,
|
|
"logps/chosen": -483.0426226265823,
|
|
"logps/rejected": -378.94041763117286,
|
|
"loss": 0.5208,
|
|
"loss/base_kto": 3.2081005573272705,
|
|
"loss/total_with_kl": 4.166040897369385,
|
|
"metrics/advantage_var": 575.3394775390625,
|
|
"regularization/advantage_var": 575.3394775390625,
|
|
"regularization/kl": 0.9579401016235352,
|
|
"rewards/chosen": 0.5378993070578273,
|
|
"rewards/margins": 0.8364122250356195,
|
|
"rewards/rejected": -0.2985129179777922,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 17.354387283325195,
|
|
"kl": 16.53875732421875,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36135237.89079563,
|
|
"logits/rejected": -36633617.62754304,
|
|
"logps/chosen": -499.41448907956317,
|
|
"logps/rejected": -381.13368055555554,
|
|
"loss": 0.5234,
|
|
"loss/base_kto": 3.1211705207824707,
|
|
"loss/total_with_kl": 4.186883449554443,
|
|
"metrics/advantage_var": 640.0679931640625,
|
|
"regularization/advantage_var": 640.0679931640625,
|
|
"regularization/kl": 1.0657131671905518,
|
|
"rewards/chosen": 0.633141480444374,
|
|
"rewards/margins": 0.9374208376859136,
|
|
"rewards/rejected": -0.30427935724153954,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 22.4042911529541,
|
|
"kl": 16.856164932250977,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35601729.28593508,
|
|
"logits/rejected": -37328983.355450235,
|
|
"logps/chosen": -483.4857032457496,
|
|
"logps/rejected": -369.04406101895734,
|
|
"loss": 0.507,
|
|
"loss/base_kto": 3.1815268993377686,
|
|
"loss/total_with_kl": 4.055867671966553,
|
|
"metrics/advantage_var": 525.12939453125,
|
|
"regularization/advantage_var": 525.12939453125,
|
|
"regularization/kl": 0.8743404746055603,
|
|
"rewards/chosen": 0.6265308786949382,
|
|
"rewards/margins": 0.8381335461916058,
|
|
"rewards/rejected": -0.21160266749666765,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 14.165505409240723,
|
|
"kl": 15.248211860656738,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33887289.06501548,
|
|
"logits/rejected": -35682013.27444795,
|
|
"logps/chosen": -477.2562403250774,
|
|
"logps/rejected": -389.7255520504732,
|
|
"loss": 0.4944,
|
|
"loss/base_kto": 3.165740489959717,
|
|
"loss/total_with_kl": 3.9552857875823975,
|
|
"metrics/advantage_var": 474.2013854980469,
|
|
"regularization/advantage_var": 474.2013854980469,
|
|
"regularization/kl": 0.7895451784133911,
|
|
"rewards/chosen": 0.5817632999951625,
|
|
"rewards/margins": 0.8701333782895977,
|
|
"rewards/rejected": -0.2883700782944351,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 22.623449325561523,
|
|
"kl": 23.995609283447266,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34935487.22891566,
|
|
"logits/rejected": -35500819.948051944,
|
|
"logps/chosen": -458.855515813253,
|
|
"logps/rejected": -381.42882508116884,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.1599528789520264,
|
|
"loss/total_with_kl": 4.1433892250061035,
|
|
"metrics/advantage_var": 590.6524658203125,
|
|
"regularization/advantage_var": 590.6524658203125,
|
|
"regularization/kl": 0.9834362864494324,
|
|
"rewards/chosen": 0.7446276653243835,
|
|
"rewards/margins": 0.8582130259628905,
|
|
"rewards/rejected": -0.11358536063850701,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 9.780498504638672,
|
|
"kl": 18.69255828857422,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34859845.09204368,
|
|
"logits/rejected": -36063350.95761382,
|
|
"logps/chosen": -486.2677457098284,
|
|
"logps/rejected": -363.1482535321821,
|
|
"loss": 0.4762,
|
|
"loss/base_kto": 3.178766965866089,
|
|
"loss/total_with_kl": 3.8092734813690186,
|
|
"metrics/advantage_var": 378.6827392578125,
|
|
"regularization/advantage_var": 378.6827392578125,
|
|
"regularization/kl": 0.6305068135261536,
|
|
"rewards/chosen": 0.6282521930760043,
|
|
"rewards/margins": 0.8451184153198599,
|
|
"rewards/rejected": -0.2168662222438555,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 131.708251953125,
|
|
"kl": 14.237939834594727,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35287872.0,
|
|
"logits/rejected": -35903532.8,
|
|
"logps/chosen": -468.3271484375,
|
|
"logps/rejected": -394.9495361328125,
|
|
"loss": 0.5186,
|
|
"loss/base_kto": 3.1408767700195312,
|
|
"loss/total_with_kl": 4.148846626281738,
|
|
"metrics/advantage_var": 605.38720703125,
|
|
"regularization/advantage_var": 605.38720703125,
|
|
"regularization/kl": 1.0079697370529175,
|
|
"rewards/chosen": 0.5717444896697998,
|
|
"rewards/margins": 0.8548405647277832,
|
|
"rewards/rejected": -0.2830960750579834,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 21.957460403442383,
|
|
"kl": 15.449542045593262,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34702918.33865815,
|
|
"logits/rejected": -36597099.25382263,
|
|
"logps/chosen": -480.67556908945687,
|
|
"logps/rejected": -388.7055141437309,
|
|
"loss": 0.4647,
|
|
"loss/base_kto": 3.1052768230438232,
|
|
"loss/total_with_kl": 3.717305898666382,
|
|
"metrics/advantage_var": 367.5849914550781,
|
|
"regularization/advantage_var": 367.5849914550781,
|
|
"regularization/kl": 0.6120290160179138,
|
|
"rewards/chosen": 0.5952093837360224,
|
|
"rewards/margins": 0.892403698395211,
|
|
"rewards/rejected": -0.29719431465918866,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 14.62973403930664,
|
|
"kl": 14.392127990722656,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35602295.68304279,
|
|
"logits/rejected": -36602777.442218795,
|
|
"logps/chosen": -497.6847266244057,
|
|
"logps/rejected": -385.8375866718028,
|
|
"loss": 0.4683,
|
|
"loss/base_kto": 3.1058084964752197,
|
|
"loss/total_with_kl": 3.7463207244873047,
|
|
"metrics/advantage_var": 384.6921081542969,
|
|
"regularization/advantage_var": 384.6921081542969,
|
|
"regularization/kl": 0.6405123472213745,
|
|
"rewards/chosen": 0.5996788254630546,
|
|
"rewards/margins": 0.8901516701790841,
|
|
"rewards/rejected": -0.29047284471602947,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 12.909411430358887,
|
|
"kl": 14.401725769042969,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35736084.10876133,
|
|
"logits/rejected": -35999147.495145634,
|
|
"logps/chosen": -505.0602341389728,
|
|
"logps/rejected": -362.46814320388347,
|
|
"loss": 0.4613,
|
|
"loss/base_kto": 3.0900819301605225,
|
|
"loss/total_with_kl": 3.690617799758911,
|
|
"metrics/advantage_var": 360.68212890625,
|
|
"regularization/advantage_var": 360.68212890625,
|
|
"regularization/kl": 0.6005356907844543,
|
|
"rewards/chosen": 0.6388769236216012,
|
|
"rewards/margins": 0.9159705578720816,
|
|
"rewards/rejected": -0.2770936342504804,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 17.458826065063477,
|
|
"kl": 13.63818073272705,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34458663.56723338,
|
|
"logits/rejected": -36154071.96208531,
|
|
"logps/chosen": -457.7695614374034,
|
|
"logps/rejected": -394.3035643759874,
|
|
"loss": 0.4696,
|
|
"loss/base_kto": 3.0972886085510254,
|
|
"loss/total_with_kl": 3.7566463947296143,
|
|
"metrics/advantage_var": 396.01080322265625,
|
|
"regularization/advantage_var": 396.01080322265625,
|
|
"regularization/kl": 0.6593579649925232,
|
|
"rewards/chosen": 0.5959498126871619,
|
|
"rewards/margins": 0.9149005441062475,
|
|
"rewards/rejected": -0.3189507314190857,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.270212173461914,
|
|
"kl": 12.883071899414062,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35798912.582701065,
|
|
"logits/rejected": -37367709.062801935,
|
|
"logps/chosen": -480.94015553869497,
|
|
"logps/rejected": -378.03930152979063,
|
|
"loss": 0.4607,
|
|
"loss/base_kto": 3.148435354232788,
|
|
"loss/total_with_kl": 3.685209035873413,
|
|
"metrics/advantage_var": 322.3865051269531,
|
|
"regularization/advantage_var": 322.3865051269531,
|
|
"regularization/kl": 0.5367735624313354,
|
|
"rewards/chosen": 0.551148248189136,
|
|
"rewards/margins": 0.8573742321820028,
|
|
"rewards/rejected": -0.3062259839928668,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.811676979064941,
|
|
"kl": 18.366003036499023,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34217687.23566879,
|
|
"logits/rejected": -36269816.14723926,
|
|
"logps/chosen": -488.60051751592357,
|
|
"logps/rejected": -358.97953412576686,
|
|
"loss": 0.4592,
|
|
"loss/base_kto": 3.086618661880493,
|
|
"loss/total_with_kl": 3.6737115383148193,
|
|
"metrics/advantage_var": 352.6083068847656,
|
|
"regularization/advantage_var": 352.6083068847656,
|
|
"regularization/kl": 0.5870928168296814,
|
|
"rewards/chosen": 0.6257616759865148,
|
|
"rewards/margins": 0.914143264226126,
|
|
"rewards/rejected": -0.2883815882396113,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 13.192625045776367,
|
|
"kl": 15.600470542907715,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34878097.38271605,
|
|
"logits/rejected": -35828706.835443035,
|
|
"logps/chosen": -515.7714120370371,
|
|
"logps/rejected": -385.5498664952532,
|
|
"loss": 0.4669,
|
|
"loss/base_kto": 3.1463685035705566,
|
|
"loss/total_with_kl": 3.7351746559143066,
|
|
"metrics/advantage_var": 353.6371765136719,
|
|
"regularization/advantage_var": 353.6371765136719,
|
|
"regularization/kl": 0.5888058543205261,
|
|
"rewards/chosen": 0.6079474555121528,
|
|
"rewards/margins": 0.8523854337496429,
|
|
"rewards/rejected": -0.2444379782374901,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 8.077712059020996,
|
|
"kl": 14.627280235290527,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34474322.69551777,
|
|
"logits/rejected": -35655987.361769356,
|
|
"logps/chosen": -472.0761205564142,
|
|
"logps/rejected": -372.5503060821485,
|
|
"loss": 0.4625,
|
|
"loss/base_kto": 3.1241071224212646,
|
|
"loss/total_with_kl": 3.6996753215789795,
|
|
"metrics/advantage_var": 345.6866760253906,
|
|
"regularization/advantage_var": 345.6866760253906,
|
|
"regularization/kl": 0.5755683183670044,
|
|
"rewards/chosen": 0.6164393240738505,
|
|
"rewards/margins": 0.8746206866176647,
|
|
"rewards/rejected": -0.2581813625438142,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 11.833416938781738,
|
|
"kl": 15.76018238067627,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35095693.18541997,
|
|
"logits/rejected": -37169199.334360555,
|
|
"logps/chosen": -474.73241877971475,
|
|
"logps/rejected": -368.6616910631741,
|
|
"loss": 0.4619,
|
|
"loss/base_kto": 3.1277101039886475,
|
|
"loss/total_with_kl": 3.694899320602417,
|
|
"metrics/advantage_var": 340.65423583984375,
|
|
"regularization/advantage_var": 340.65423583984375,
|
|
"regularization/kl": 0.5671892762184143,
|
|
"rewards/chosen": 0.6180357048893126,
|
|
"rewards/margins": 0.8669078946388672,
|
|
"rewards/rejected": -0.2488721897495546,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.888648509979248,
|
|
"kl": 17.30721092224121,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -33943859.36704731,
|
|
"logits/rejected": -35966067.142428786,
|
|
"logps/chosen": -471.4657422512235,
|
|
"logps/rejected": -362.8520193028486,
|
|
"loss": 0.4554,
|
|
"loss/base_kto": 3.0811221599578857,
|
|
"loss/total_with_kl": 3.643476963043213,
|
|
"metrics/advantage_var": 337.75054931640625,
|
|
"regularization/advantage_var": 337.75054931640625,
|
|
"regularization/kl": 0.562354564666748,
|
|
"rewards/chosen": 0.6325444132818873,
|
|
"rewards/margins": 0.8998226927822738,
|
|
"rewards/rejected": -0.2672782795003865,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 10.333745002746582,
|
|
"kl": 13.518071174621582,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34612914.15673981,
|
|
"logits/rejected": -35654579.43925234,
|
|
"logps/chosen": -482.26400862068965,
|
|
"logps/rejected": -382.2936137071651,
|
|
"loss": 0.4715,
|
|
"loss/base_kto": 3.1216957569122314,
|
|
"loss/total_with_kl": 3.7717549800872803,
|
|
"metrics/advantage_var": 390.42608642578125,
|
|
"regularization/advantage_var": 390.42608642578125,
|
|
"regularization/kl": 0.6500594019889832,
|
|
"rewards/chosen": 0.48953883251800057,
|
|
"rewards/margins": 0.8870594266543664,
|
|
"rewards/rejected": -0.39752059413636587,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 10.766597747802734,
|
|
"kl": 13.293540954589844,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36287738.87211094,
|
|
"logits/rejected": -36835013.17274168,
|
|
"logps/chosen": -493.5424210323575,
|
|
"logps/rejected": -395.0152040412044,
|
|
"loss": 0.4743,
|
|
"loss/base_kto": 3.1135079860687256,
|
|
"loss/total_with_kl": 3.794252395629883,
|
|
"metrics/advantage_var": 408.8554382324219,
|
|
"regularization/advantage_var": 408.8554382324219,
|
|
"regularization/kl": 0.6807443499565125,
|
|
"rewards/chosen": 0.5670461786913762,
|
|
"rewards/margins": 0.893886877127019,
|
|
"rewards/rejected": -0.32684069843564284,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 17.946382522583008,
|
|
"kl": 17.167036056518555,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33566674.98901099,
|
|
"logits/rejected": -35586831.52721617,
|
|
"logps/chosen": -486.8195643642072,
|
|
"logps/rejected": -389.01693720839813,
|
|
"loss": 0.4692,
|
|
"loss/base_kto": 3.1354446411132812,
|
|
"loss/total_with_kl": 3.7535560131073,
|
|
"metrics/advantage_var": 371.2381591796875,
|
|
"regularization/advantage_var": 371.2381591796875,
|
|
"regularization/kl": 0.6181115508079529,
|
|
"rewards/chosen": 0.5946457891299549,
|
|
"rewards/margins": 0.8752973740696297,
|
|
"rewards/rejected": -0.28065158493967485,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 10.882978439331055,
|
|
"kl": 15.592175483703613,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33698434.899676375,
|
|
"logits/rejected": -34695891.91540785,
|
|
"logps/chosen": -461.0573927993527,
|
|
"logps/rejected": -375.73966200906347,
|
|
"loss": 0.4647,
|
|
"loss/base_kto": 3.1134324073791504,
|
|
"loss/total_with_kl": 3.717601776123047,
|
|
"metrics/advantage_var": 362.8645324707031,
|
|
"regularization/advantage_var": 362.8645324707031,
|
|
"regularization/kl": 0.6041694283485413,
|
|
"rewards/chosen": 0.5645856641257079,
|
|
"rewards/margins": 0.8674073737960368,
|
|
"rewards/rejected": -0.302821709670329,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 9.397636413574219,
|
|
"kl": 16.236923217773438,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34346672.65497076,
|
|
"logits/rejected": -36113016.26845638,
|
|
"logps/chosen": -455.4813139619883,
|
|
"logps/rejected": -363.28009647651004,
|
|
"loss": 0.4539,
|
|
"loss/base_kto": 3.0974628925323486,
|
|
"loss/total_with_kl": 3.631000518798828,
|
|
"metrics/advantage_var": 320.4429626464844,
|
|
"regularization/advantage_var": 320.4429626464844,
|
|
"regularization/kl": 0.5335375666618347,
|
|
"rewards/chosen": 0.6008654588844344,
|
|
"rewards/margins": 0.901654466288941,
|
|
"rewards/rejected": -0.3007890074045066,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 10.22039794921875,
|
|
"kl": 20.599092483520508,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35185682.197092086,
|
|
"logits/rejected": -35385703.40695915,
|
|
"logps/chosen": -475.23485460420034,
|
|
"logps/rejected": -372.04297465960667,
|
|
"loss": 0.456,
|
|
"loss/base_kto": 3.088881731033325,
|
|
"loss/total_with_kl": 3.6476452350616455,
|
|
"metrics/advantage_var": 335.59381103515625,
|
|
"regularization/advantage_var": 335.59381103515625,
|
|
"regularization/kl": 0.5587636828422546,
|
|
"rewards/chosen": 0.6566993331292912,
|
|
"rewards/margins": 0.8689126618038846,
|
|
"rewards/rejected": -0.21221332867459342,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 14.697084426879883,
|
|
"kl": 16.490514755249023,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35019740.152207,
|
|
"logits/rejected": -36298107.68539326,
|
|
"logps/chosen": -481.2613203957382,
|
|
"logps/rejected": -378.3217044542536,
|
|
"loss": 0.465,
|
|
"loss/base_kto": 3.1004292964935303,
|
|
"loss/total_with_kl": 3.7196145057678223,
|
|
"metrics/advantage_var": 371.8828125,
|
|
"regularization/advantage_var": 371.8828125,
|
|
"regularization/kl": 0.6191849112510681,
|
|
"rewards/chosen": 0.6302919431364156,
|
|
"rewards/margins": 0.9196072050996981,
|
|
"rewards/rejected": -0.2893152619632825,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 14.012263298034668,
|
|
"kl": 16.660383224487305,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -36057626.062397376,
|
|
"logits/rejected": -36866725.57973174,
|
|
"logps/chosen": -478.07784277504106,
|
|
"logps/rejected": -379.0761456780924,
|
|
"loss": 0.4673,
|
|
"loss/base_kto": 3.1529088020324707,
|
|
"loss/total_with_kl": 3.738218069076538,
|
|
"metrics/advantage_var": 351.5369567871094,
|
|
"regularization/advantage_var": 351.5369567871094,
|
|
"regularization/kl": 0.5853090286254883,
|
|
"rewards/chosen": 0.5958792650445146,
|
|
"rewards/margins": 0.8451927787193003,
|
|
"rewards/rejected": -0.24931351367478577,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.914913177490234,
|
|
"kl": 15.437067985534668,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34886832.98765432,
|
|
"logits/rejected": -35992951.898734175,
|
|
"logps/chosen": -502.68315972222223,
|
|
"logps/rejected": -365.6356803797468,
|
|
"loss": 0.4678,
|
|
"loss/base_kto": 3.106647491455078,
|
|
"loss/total_with_kl": 3.7421205043792725,
|
|
"metrics/advantage_var": 381.6656188964844,
|
|
"regularization/advantage_var": 381.6656188964844,
|
|
"regularization/kl": 0.6354732513427734,
|
|
"rewards/chosen": 0.6248187783323689,
|
|
"rewards/margins": 0.8890638719556629,
|
|
"rewards/rejected": -0.2642450936232941,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 8.364889144897461,
|
|
"kl": 15.678446769714355,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34099149.19689923,
|
|
"logits/rejected": -36949003.28818898,
|
|
"logps/chosen": -481.9557170542636,
|
|
"logps/rejected": -364.8339566929134,
|
|
"loss": 0.4611,
|
|
"loss/base_kto": 3.095292329788208,
|
|
"loss/total_with_kl": 3.688654661178589,
|
|
"metrics/advantage_var": 356.37371826171875,
|
|
"regularization/advantage_var": 356.37371826171875,
|
|
"regularization/kl": 0.5933622717857361,
|
|
"rewards/chosen": 0.6045946697856105,
|
|
"rewards/margins": 0.9088821972754604,
|
|
"rewards/rejected": -0.3042875274898499,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 11.69877815246582,
|
|
"kl": 16.17605972290039,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -33835150.222222224,
|
|
"logits/rejected": -35831328.405063294,
|
|
"logps/chosen": -473.37229938271605,
|
|
"logps/rejected": -359.07748219936707,
|
|
"loss": 0.4681,
|
|
"loss/base_kto": 3.1174633502960205,
|
|
"loss/total_with_kl": 3.744802236557007,
|
|
"metrics/advantage_var": 376.7802429199219,
|
|
"regularization/advantage_var": 376.7802429199219,
|
|
"regularization/kl": 0.6273390650749207,
|
|
"rewards/chosen": 0.6363465579939477,
|
|
"rewards/margins": 0.8976437115151354,
|
|
"rewards/rejected": -0.2612971535211877,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 15.175222396850586,
|
|
"kl": 17.070270538330078,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35250530.461538464,
|
|
"logits/rejected": -36317056.0,
|
|
"logps/chosen": -477.95658052884613,
|
|
"logps/rejected": -381.0267244664634,
|
|
"loss": 0.4632,
|
|
"loss/base_kto": 3.1355667114257812,
|
|
"loss/total_with_kl": 3.7057037353515625,
|
|
"metrics/advantage_var": 342.4247741699219,
|
|
"regularization/advantage_var": 342.4247741699219,
|
|
"regularization/kl": 0.5701372027397156,
|
|
"rewards/chosen": 0.6318840613731971,
|
|
"rewards/margins": 0.8648185944691384,
|
|
"rewards/rejected": -0.2329345330959413,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 11.70081615447998,
|
|
"kl": 13.312631607055664,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34734253.88679245,
|
|
"logits/rejected": -36847981.71428572,
|
|
"logps/chosen": -498.66838639937106,
|
|
"logps/rejected": -359.1365003881988,
|
|
"loss": 0.4611,
|
|
"loss/base_kto": 3.089224338531494,
|
|
"loss/total_with_kl": 3.689150333404541,
|
|
"metrics/advantage_var": 360.3158874511719,
|
|
"regularization/advantage_var": 360.3158874511719,
|
|
"regularization/kl": 0.5999259948730469,
|
|
"rewards/chosen": 0.5956237601034297,
|
|
"rewards/margins": 0.8988537140351551,
|
|
"rewards/rejected": -0.3032299539317255,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 9.287192344665527,
|
|
"kl": 11.785021781921387,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34309818.821374044,
|
|
"logits/rejected": -34383580.3648,
|
|
"logps/chosen": -490.9625,
|
|
"logps/rejected": -364.72515,
|
|
"loss": 0.4588,
|
|
"loss/base_kto": 3.063136577606201,
|
|
"loss/total_with_kl": 3.670264482498169,
|
|
"metrics/advantage_var": 364.6414489746094,
|
|
"regularization/advantage_var": 364.6414489746094,
|
|
"regularization/kl": 0.6071280837059021,
|
|
"rewards/chosen": 0.6333160633349236,
|
|
"rewards/margins": 0.9518114734911736,
|
|
"rewards/rejected": -0.31849541015625,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 15.305978775024414,
|
|
"kl": 11.813769340515137,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34512877.239694655,
|
|
"logits/rejected": -35414599.2704,
|
|
"logps/chosen": -480.8729961832061,
|
|
"logps/rejected": -361.2996,
|
|
"loss": 0.4523,
|
|
"loss/base_kto": 3.102526903152466,
|
|
"loss/total_with_kl": 3.618056535720825,
|
|
"metrics/advantage_var": 309.6274719238281,
|
|
"regularization/advantage_var": 309.6274719238281,
|
|
"regularization/kl": 0.5155297517776489,
|
|
"rewards/chosen": 0.5750778081762882,
|
|
"rewards/margins": 0.8881732427466007,
|
|
"rewards/rejected": -0.3130954345703125,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.634213447570801,
|
|
"kl": 14.661700248718262,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35294220.449848026,
|
|
"logits/rejected": -36244944.25723473,
|
|
"logps/chosen": -447.7244490881459,
|
|
"logps/rejected": -356.9380526527331,
|
|
"loss": 0.4587,
|
|
"loss/base_kto": 3.1317965984344482,
|
|
"loss/total_with_kl": 3.669286012649536,
|
|
"metrics/advantage_var": 322.816650390625,
|
|
"regularization/advantage_var": 322.816650390625,
|
|
"regularization/kl": 0.5374897122383118,
|
|
"rewards/chosen": 0.5784724737010828,
|
|
"rewards/margins": 0.8650531390196117,
|
|
"rewards/rejected": -0.28658066531852894,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 11.300702095031738,
|
|
"kl": 16.711589813232422,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -35452267.85007496,
|
|
"logits/rejected": -37956980.51549755,
|
|
"logps/chosen": -477.5771645427286,
|
|
"logps/rejected": -383.44810358890703,
|
|
"loss": 0.4608,
|
|
"loss/base_kto": 3.1482601165771484,
|
|
"loss/total_with_kl": 3.6863152980804443,
|
|
"metrics/advantage_var": 323.15625,
|
|
"regularization/advantage_var": 323.15625,
|
|
"regularization/kl": 0.5380551218986511,
|
|
"rewards/chosen": 0.6162983724202352,
|
|
"rewards/margins": 0.8408679700223203,
|
|
"rewards/rejected": -0.22456959760208503,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 11.286430358886719,
|
|
"kl": 17.403745651245117,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35532519.74736842,
|
|
"logits/rejected": -35742575.141463414,
|
|
"logps/chosen": -448.54459586466163,
|
|
"logps/rejected": -392.8595782520325,
|
|
"loss": 0.4674,
|
|
"loss/base_kto": 3.1981921195983887,
|
|
"loss/total_with_kl": 3.7389843463897705,
|
|
"metrics/advantage_var": 324.8004455566406,
|
|
"regularization/advantage_var": 324.8004455566406,
|
|
"regularization/kl": 0.54079270362854,
|
|
"rewards/chosen": 0.5928094161184211,
|
|
"rewards/margins": 0.7968746924736994,
|
|
"rewards/rejected": -0.2040652763552782,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 11.632631301879883,
|
|
"kl": 18.090106964111328,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34502069.937106915,
|
|
"logits/rejected": -35282708.67080745,
|
|
"logps/chosen": -487.7014544025157,
|
|
"logps/rejected": -389.0685413431677,
|
|
"loss": 0.4746,
|
|
"loss/base_kto": 3.142620086669922,
|
|
"loss/total_with_kl": 3.7968201637268066,
|
|
"metrics/advantage_var": 392.9130859375,
|
|
"regularization/advantage_var": 392.9130859375,
|
|
"regularization/kl": 0.6542002558708191,
|
|
"rewards/chosen": 0.6386708673441185,
|
|
"rewards/margins": 0.8350709825527147,
|
|
"rewards/rejected": -0.19640011520859618,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 13.436630249023438,
|
|
"kl": 18.01979637145996,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -33245145.05229794,
|
|
"logits/rejected": -34113883.118644066,
|
|
"logps/chosen": -484.50168383518223,
|
|
"logps/rejected": -369.4667035824345,
|
|
"loss": 0.4719,
|
|
"loss/base_kto": 3.071901321411133,
|
|
"loss/total_with_kl": 3.775254011154175,
|
|
"metrics/advantage_var": 422.4339294433594,
|
|
"regularization/advantage_var": 422.4339294433594,
|
|
"regularization/kl": 0.7033524513244629,
|
|
"rewards/chosen": 0.6746830040981577,
|
|
"rewards/margins": 0.9257238933267786,
|
|
"rewards/rejected": -0.25104088922862094,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 11.870504379272461,
|
|
"kl": 19.13331413269043,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -33964642.523659304,
|
|
"logits/rejected": -36241465.06501548,
|
|
"logps/chosen": -485.45613170347,
|
|
"logps/rejected": -373.0215267027864,
|
|
"loss": 0.4538,
|
|
"loss/base_kto": 3.1138417720794678,
|
|
"loss/total_with_kl": 3.6300301551818848,
|
|
"metrics/advantage_var": 310.0228576660156,
|
|
"regularization/advantage_var": 310.0228576660156,
|
|
"regularization/kl": 0.5161880850791931,
|
|
"rewards/chosen": 0.6674954868641562,
|
|
"rewards/margins": 0.8776987225966687,
|
|
"rewards/rejected": -0.21020323573251257,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 9.92672061920166,
|
|
"kl": 15.61523723602295,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34283768.91076923,
|
|
"logits/rejected": -35964693.942857146,
|
|
"logps/chosen": -472.1594230769231,
|
|
"logps/rejected": -378.12564484126983,
|
|
"loss": 0.4605,
|
|
"loss/base_kto": 3.120116710662842,
|
|
"loss/total_with_kl": 3.68363356590271,
|
|
"metrics/advantage_var": 338.44842529296875,
|
|
"regularization/advantage_var": 338.44842529296875,
|
|
"regularization/kl": 0.5635165572166443,
|
|
"rewards/chosen": 0.6128104341947115,
|
|
"rewards/margins": 0.8644788980076454,
|
|
"rewards/rejected": -0.251668463812934,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 35.8725471496582,
|
|
"kl": 19.759702682495117,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -33160753.118699186,
|
|
"logits/rejected": -35798134.56842105,
|
|
"logps/chosen": -484.73460365853657,
|
|
"logps/rejected": -365.2447838345865,
|
|
"loss": 0.4687,
|
|
"loss/base_kto": 3.0961015224456787,
|
|
"loss/total_with_kl": 3.749312162399292,
|
|
"metrics/advantage_var": 392.3187255859375,
|
|
"regularization/advantage_var": 392.3187255859375,
|
|
"regularization/kl": 0.6532106399536133,
|
|
"rewards/chosen": 0.6879379644626524,
|
|
"rewards/margins": 0.9006296515661534,
|
|
"rewards/rejected": -0.21269168710350095,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 10.404247283935547,
|
|
"kl": 17.5299015045166,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35088032.356120825,
|
|
"logits/rejected": -37030394.49462365,
|
|
"logps/chosen": -463.7991852146264,
|
|
"logps/rejected": -378.0746687788018,
|
|
"loss": 0.4547,
|
|
"loss/base_kto": 3.104525089263916,
|
|
"loss/total_with_kl": 3.637596845626831,
|
|
"metrics/advantage_var": 320.1632385253906,
|
|
"regularization/advantage_var": 320.1632385253906,
|
|
"regularization/kl": 0.5330718159675598,
|
|
"rewards/chosen": 0.6244717402374801,
|
|
"rewards/margins": 0.8903324428224455,
|
|
"rewards/rejected": -0.2658607025849654,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 14.288045883178711,
|
|
"kl": 17.466943740844727,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -36059022.402535655,
|
|
"logits/rejected": -36814343.10015408,
|
|
"logps/chosen": -508.334587955626,
|
|
"logps/rejected": -351.3088405238829,
|
|
"loss": 0.4702,
|
|
"loss/base_kto": 3.1115715503692627,
|
|
"loss/total_with_kl": 3.7619361877441406,
|
|
"metrics/advantage_var": 390.6095275878906,
|
|
"regularization/advantage_var": 390.6095275878906,
|
|
"regularization/kl": 0.650364875793457,
|
|
"rewards/chosen": 0.6279828393515996,
|
|
"rewards/margins": 0.8883667627102523,
|
|
"rewards/rejected": -0.26038392335865274,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 19.08237075805664,
|
|
"kl": 19.452606201171875,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -36031179.02053712,
|
|
"logits/rejected": -36443495.270479135,
|
|
"logps/chosen": -487.4218996840442,
|
|
"logps/rejected": -389.8936920401855,
|
|
"loss": 0.4667,
|
|
"loss/base_kto": 3.090646743774414,
|
|
"loss/total_with_kl": 3.7336056232452393,
|
|
"metrics/advantage_var": 386.1615905761719,
|
|
"regularization/advantage_var": 386.1615905761719,
|
|
"regularization/kl": 0.6429590582847595,
|
|
"rewards/chosen": 0.6275976099674171,
|
|
"rewards/margins": 0.8866167570827088,
|
|
"rewards/rejected": -0.2590191471152917,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_loss": 0.4995092036192899,
|
|
"train_runtime": 10988.4166,
|
|
"train_samples_per_second": 13.489,
|
|
"train_steps_per_second": 0.211
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|