2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 24.171916961669922,
|
|
"kl": 17.971378326416016,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -35316779.70731708,
|
|
"logits/rejected": -36768866.461538464,
|
|
"logps/chosen": -497.724228277439,
|
|
"logps/rejected": -363.51056690705127,
|
|
"loss": 0.6047,
|
|
"loss/base_kto": 3.8854215145111084,
|
|
"metrics/advantage_var": 204.70074462890625,
|
|
"regularization/lipschitz_norm": 983.5292358398438,
|
|
"regularization/w1": 0.952056348323822,
|
|
"rewards/chosen": 0.28100246336401963,
|
|
"rewards/margins": 0.11808828505372315,
|
|
"rewards/rejected": 0.16291417831029648,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 22.6082820892334,
|
|
"kl": 14.165916442871094,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37087235.13629403,
|
|
"logits/rejected": -37193473.22488038,
|
|
"logps/chosen": -483.79216117917304,
|
|
"logps/rejected": -404.0459778708134,
|
|
"loss": 0.6051,
|
|
"loss/base_kto": 3.9289321899414062,
|
|
"metrics/advantage_var": 192.48802185058594,
|
|
"regularization/lipschitz_norm": 941.6502075195312,
|
|
"regularization/w1": 0.911517322063446,
|
|
"rewards/chosen": 0.13989972849169696,
|
|
"rewards/margins": 0.07407873944570663,
|
|
"rewards/rejected": 0.06582098904599033,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 26.396751403808594,
|
|
"kl": 10.408247947692871,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35812877.32924962,
|
|
"logits/rejected": -37073189.97129186,
|
|
"logps/chosen": -493.8341787901991,
|
|
"logps/rejected": -388.9180123604466,
|
|
"loss": 0.5946,
|
|
"loss/base_kto": 3.8426125049591064,
|
|
"metrics/advantage_var": 201.38356018066406,
|
|
"regularization/lipschitz_norm": 944.7931518554688,
|
|
"regularization/w1": 0.9145596623420715,
|
|
"rewards/chosen": 0.19716486558432236,
|
|
"rewards/margins": 0.1451017434540455,
|
|
"rewards/rejected": 0.05206312213027686,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 21.040809631347656,
|
|
"kl": 3.604231357574463,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37383490.82033898,
|
|
"logits/rejected": -37841154.22608696,
|
|
"logps/chosen": -490.29480932203387,
|
|
"logps/rejected": -394.2195652173913,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.840106248855591,
|
|
"metrics/advantage_var": 192.21023559570312,
|
|
"regularization/lipschitz_norm": 967.9828491210938,
|
|
"regularization/w1": 0.9370073676109314,
|
|
"rewards/chosen": -0.0987275980286679,
|
|
"rewards/margins": 0.12818333310244623,
|
|
"rewards/rejected": -0.22691093113111413,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 22.586702346801758,
|
|
"kl": 7.076615333557129,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35031971.77639752,
|
|
"logits/rejected": -36753507.82389937,
|
|
"logps/chosen": -482.4668575310559,
|
|
"logps/rejected": -354.69388757861634,
|
|
"loss": 0.6005,
|
|
"loss/base_kto": 3.8660409450531006,
|
|
"metrics/advantage_var": 210.2734832763672,
|
|
"regularization/lipschitz_norm": 969.1918334960938,
|
|
"regularization/w1": 0.9381775856018066,
|
|
"rewards/chosen": 0.018050518095123102,
|
|
"rewards/margins": 0.13388869489439345,
|
|
"rewards/rejected": -0.11583817679927035,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 24.313669204711914,
|
|
"kl": 13.761123657226562,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36796998.33865815,
|
|
"logits/rejected": -36799199.90214067,
|
|
"logps/chosen": -469.5819688498403,
|
|
"logps/rejected": -374.8200974770642,
|
|
"loss": 0.6077,
|
|
"loss/base_kto": 3.8909480571746826,
|
|
"metrics/advantage_var": 223.26220703125,
|
|
"regularization/lipschitz_norm": 1002.9113159179688,
|
|
"regularization/w1": 0.9708181619644165,
|
|
"rewards/chosen": 0.21156050221988568,
|
|
"rewards/margins": 0.09433169383145272,
|
|
"rewards/rejected": 0.11722880838843296,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 21.588844299316406,
|
|
"kl": 9.700366020202637,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36744837.53204173,
|
|
"logits/rejected": -38196094.52873563,
|
|
"logps/chosen": -496.6974664679583,
|
|
"logps/rejected": -368.6950687602627,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.799974203109741,
|
|
"metrics/advantage_var": 212.4312744140625,
|
|
"regularization/lipschitz_norm": 985.76953125,
|
|
"regularization/w1": 0.9542250037193298,
|
|
"rewards/chosen": 0.17170122945361982,
|
|
"rewards/margins": 0.2000509132434039,
|
|
"rewards/rejected": -0.028349683789784096,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 17.44001007080078,
|
|
"kl": 4.787113666534424,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36035446.15384615,
|
|
"logits/rejected": -36212767.219512194,
|
|
"logps/chosen": -477.42322716346155,
|
|
"logps/rejected": -349.8604706554878,
|
|
"loss": 0.5934,
|
|
"loss/base_kto": 3.8393142223358154,
|
|
"metrics/advantage_var": 205.9862060546875,
|
|
"regularization/lipschitz_norm": 938.0133056640625,
|
|
"regularization/w1": 0.9079969525337219,
|
|
"rewards/chosen": 0.06676110854515663,
|
|
"rewards/margins": 0.1606005620330181,
|
|
"rewards/rejected": -0.09383945348786145,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 18.280359268188477,
|
|
"kl": 1.6990368366241455,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35199759.52721617,
|
|
"logits/rejected": -36141772.96075353,
|
|
"logps/chosen": -456.05355754276826,
|
|
"logps/rejected": -362.4076236263736,
|
|
"loss": 0.5899,
|
|
"loss/base_kto": 3.8271141052246094,
|
|
"metrics/advantage_var": 199.59747314453125,
|
|
"regularization/lipschitz_norm": 921.5966796875,
|
|
"regularization/w1": 0.8921055197715759,
|
|
"rewards/chosen": -0.10501613587280084,
|
|
"rewards/margins": 0.1806835857225681,
|
|
"rewards/rejected": -0.28569972159536894,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 19.40365219116211,
|
|
"kl": 9.957293510437012,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -34833159.70839936,
|
|
"logits/rejected": -37904980.41294299,
|
|
"logps/chosen": -497.16536251980983,
|
|
"logps/rejected": -373.4144356702619,
|
|
"loss": 0.5977,
|
|
"loss/base_kto": 3.7903313636779785,
|
|
"metrics/advantage_var": 230.1645050048828,
|
|
"regularization/lipschitz_norm": 1023.9474487304688,
|
|
"regularization/w1": 0.9911810159683228,
|
|
"rewards/chosen": 0.16381330535453398,
|
|
"rewards/margins": 0.2036897788739283,
|
|
"rewards/rejected": -0.03987647351939432,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 24.024091720581055,
|
|
"kl": 6.649497985839844,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36357843.109004736,
|
|
"logits/rejected": -36603995.79598145,
|
|
"logps/chosen": -480.86868088467617,
|
|
"logps/rejected": -375.7693199381762,
|
|
"loss": 0.6005,
|
|
"loss/base_kto": 3.7809770107269287,
|
|
"metrics/advantage_var": 276.2771911621094,
|
|
"regularization/lipschitz_norm": 1056.7017822265625,
|
|
"regularization/w1": 1.0228872299194336,
|
|
"rewards/chosen": 0.07966904240950212,
|
|
"rewards/margins": 0.2167478833226409,
|
|
"rewards/rejected": -0.13707884091313877,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 24.00461196899414,
|
|
"kl": 8.452391624450684,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36556822.22635659,
|
|
"logits/rejected": -36657490.64566929,
|
|
"logps/chosen": -473.9613856589147,
|
|
"logps/rejected": -386.12573818897636,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.76371169090271,
|
|
"metrics/advantage_var": 254.5139923095703,
|
|
"regularization/lipschitz_norm": 1058.958740234375,
|
|
"regularization/w1": 1.0250719785690308,
|
|
"rewards/chosen": 0.09946464124576065,
|
|
"rewards/margins": 0.22167490207253232,
|
|
"rewards/rejected": -0.12221026082677165,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 30.11318588256836,
|
|
"kl": 13.7765531539917,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35608964.93768546,
|
|
"logits/rejected": -37566044.93729373,
|
|
"logps/chosen": -479.20850333827894,
|
|
"logps/rejected": -371.1122112211221,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.685382843017578,
|
|
"metrics/advantage_var": 293.2428894042969,
|
|
"regularization/lipschitz_norm": 1121.0953369140625,
|
|
"regularization/w1": 1.0852203369140625,
|
|
"rewards/chosen": 0.3413410582952754,
|
|
"rewards/margins": 0.2959683660234542,
|
|
"rewards/rejected": 0.045372692271821176,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 27.58708381652832,
|
|
"kl": 22.29041862487793,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37715201.9306184,
|
|
"logits/rejected": -39577872.1815235,
|
|
"logps/chosen": -489.99769042232276,
|
|
"logps/rejected": -354.0433549432739,
|
|
"loss": 0.6002,
|
|
"loss/base_kto": 3.7528398036956787,
|
|
"metrics/advantage_var": 287.2712097167969,
|
|
"regularization/lipschitz_norm": 1083.8204345703125,
|
|
"regularization/w1": 1.0491383075714111,
|
|
"rewards/chosen": 0.3891103062694429,
|
|
"rewards/margins": 0.2284970345159988,
|
|
"rewards/rejected": 0.1606132717534441,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 18.10953712463379,
|
|
"kl": 22.626684188842773,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36378617.52922591,
|
|
"logits/rejected": -37791354.65842349,
|
|
"logps/chosen": -491.140304107425,
|
|
"logps/rejected": -381.2754781684699,
|
|
"loss": 0.5995,
|
|
"loss/base_kto": 3.845449924468994,
|
|
"metrics/advantage_var": 213.772216796875,
|
|
"regularization/lipschitz_norm": 982.02880859375,
|
|
"regularization/w1": 0.9506039023399353,
|
|
"rewards/chosen": 0.3383839443019969,
|
|
"rewards/margins": 0.13321049314219688,
|
|
"rewards/rejected": 0.20517345115980004,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 25.496047973632812,
|
|
"kl": 14.383989334106445,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37659781.835866265,
|
|
"logits/rejected": -38620067.80707396,
|
|
"logps/chosen": -494.45825417933133,
|
|
"logps/rejected": -358.76914188102893,
|
|
"loss": 0.584,
|
|
"loss/base_kto": 3.7954728603363037,
|
|
"metrics/advantage_var": 201.11444091796875,
|
|
"regularization/lipschitz_norm": 905.8062744140625,
|
|
"regularization/w1": 0.8768205046653748,
|
|
"rewards/chosen": 0.24768414250985468,
|
|
"rewards/margins": 0.17627670173554424,
|
|
"rewards/rejected": 0.07140744077431044,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 22.850505828857422,
|
|
"kl": 9.190020561218262,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36118437.97802198,
|
|
"logits/rejected": -37721006.7807154,
|
|
"logps/chosen": -476.0750588697017,
|
|
"logps/rejected": -363.1827614696734,
|
|
"loss": 0.5889,
|
|
"loss/base_kto": 3.759268283843994,
|
|
"metrics/advantage_var": 211.2550048828125,
|
|
"regularization/lipschitz_norm": 983.0006103515625,
|
|
"regularization/w1": 0.951544463634491,
|
|
"rewards/chosen": 0.12163652692522321,
|
|
"rewards/margins": 0.23103632542377733,
|
|
"rewards/rejected": -0.10939979849855413,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 21.94005012512207,
|
|
"kl": 14.6563081741333,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36833775.43297381,
|
|
"logits/rejected": -37856299.81616482,
|
|
"logps/chosen": -486.2754718798151,
|
|
"logps/rejected": -386.76116778922346,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.7516915798187256,
|
|
"metrics/advantage_var": 250.81607055664062,
|
|
"regularization/lipschitz_norm": 1073.2459716796875,
|
|
"regularization/w1": 1.038901925086975,
|
|
"rewards/chosen": 0.2067007761339561,
|
|
"rewards/margins": 0.20732436276141453,
|
|
"rewards/rejected": -0.0006235866274584303,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 25.312530517578125,
|
|
"kl": 15.028098106384277,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35418189.45839637,
|
|
"logits/rejected": -38815198.08723748,
|
|
"logps/chosen": -501.94752269288955,
|
|
"logps/rejected": -371.9616316639742,
|
|
"loss": 0.5972,
|
|
"loss/base_kto": 3.7215523719787598,
|
|
"metrics/advantage_var": 279.48394775390625,
|
|
"regularization/lipschitz_norm": 1090.6280517578125,
|
|
"regularization/w1": 1.0557279586791992,
|
|
"rewards/chosen": 0.30185509016582357,
|
|
"rewards/margins": 0.23913481142743503,
|
|
"rewards/rejected": 0.06272027873838854,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 21.098722457885742,
|
|
"kl": 15.414970397949219,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36133701.67340591,
|
|
"logits/rejected": -37539753.19309262,
|
|
"logps/chosen": -461.15697900466563,
|
|
"logps/rejected": -361.7773253532182,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.795140504837036,
|
|
"metrics/advantage_var": 254.8007049560547,
|
|
"regularization/lipschitz_norm": 1016.0383911132812,
|
|
"regularization/w1": 0.9835252165794373,
|
|
"rewards/chosen": 0.2212227543998469,
|
|
"rewards/margins": 0.17877199477564715,
|
|
"rewards/rejected": 0.04245075962419974,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 26.532087326049805,
|
|
"kl": 10.874388694763184,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35524204.45320197,
|
|
"logits/rejected": -37241613.353204176,
|
|
"logps/chosen": -472.52324507389164,
|
|
"logps/rejected": -351.55993852459017,
|
|
"loss": 0.5838,
|
|
"loss/base_kto": 3.7438149452209473,
|
|
"metrics/advantage_var": 227.2717742919922,
|
|
"regularization/lipschitz_norm": 956.939453125,
|
|
"regularization/w1": 0.9263173937797546,
|
|
"rewards/chosen": 0.21551162895114942,
|
|
"rewards/margins": 0.22224590135796485,
|
|
"rewards/rejected": -0.006734272406815417,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 27.539342880249023,
|
|
"kl": 6.9973602294921875,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37273823.41818182,
|
|
"logits/rejected": -37989006.03870968,
|
|
"logps/chosen": -509.3034564393939,
|
|
"logps/rejected": -372.19339717741934,
|
|
"loss": 0.5934,
|
|
"loss/base_kto": 3.7677838802337646,
|
|
"metrics/advantage_var": 236.2348175048828,
|
|
"regularization/lipschitz_norm": 1011.4757690429688,
|
|
"regularization/w1": 0.979108452796936,
|
|
"rewards/chosen": 0.08697488958185369,
|
|
"rewards/margins": 0.22105295846888745,
|
|
"rewards/rejected": -0.13407806888703377,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 22.417484283447266,
|
|
"kl": 10.195749282836914,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -37403931.01507538,
|
|
"logits/rejected": -38144776.62079063,
|
|
"logps/chosen": -471.87646566164153,
|
|
"logps/rejected": -401.33437042459735,
|
|
"loss": 0.5911,
|
|
"loss/base_kto": 3.7370898723602295,
|
|
"metrics/advantage_var": 287.5673828125,
|
|
"regularization/lipschitz_norm": 1024.3519287109375,
|
|
"regularization/w1": 0.9915726780891418,
|
|
"rewards/chosen": 0.08149675148815366,
|
|
"rewards/margins": 0.22253619628449106,
|
|
"rewards/rejected": -0.1410394447963374,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 20.285343170166016,
|
|
"kl": 8.333754539489746,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36534663.29738059,
|
|
"logits/rejected": -37380453.02060222,
|
|
"logps/chosen": -496.8182781201849,
|
|
"logps/rejected": -372.7764461172742,
|
|
"loss": 0.5937,
|
|
"loss/base_kto": 3.729773759841919,
|
|
"metrics/advantage_var": 263.1363220214844,
|
|
"regularization/lipschitz_norm": 1053.1697998046875,
|
|
"regularization/w1": 1.0194681882858276,
|
|
"rewards/chosen": 0.08392554989948478,
|
|
"rewards/margins": 0.25394270164894,
|
|
"rewards/rejected": -0.17001715174945523,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 22.549976348876953,
|
|
"kl": 7.499153137207031,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -34990649.642384104,
|
|
"logits/rejected": -36608830.10650887,
|
|
"logps/chosen": -490.3294701986755,
|
|
"logps/rejected": -375.35470599112426,
|
|
"loss": 0.5903,
|
|
"loss/base_kto": 3.681213855743408,
|
|
"metrics/advantage_var": 266.59619140625,
|
|
"regularization/lipschitz_norm": 1075.8951416015625,
|
|
"regularization/w1": 1.0414663553237915,
|
|
"rewards/chosen": 0.022774854243196398,
|
|
"rewards/margins": 0.27699002700519254,
|
|
"rewards/rejected": -0.2542151727619961,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 16.535247802734375,
|
|
"kl": 14.017804145812988,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36697649.34939759,
|
|
"logits/rejected": -36691559.064935066,
|
|
"logps/chosen": -508.0226374246988,
|
|
"logps/rejected": -376.5061891233766,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.684291124343872,
|
|
"metrics/advantage_var": 312.0481872558594,
|
|
"regularization/lipschitz_norm": 1130.4091796875,
|
|
"regularization/w1": 1.0942360162734985,
|
|
"rewards/chosen": 0.28289523756647683,
|
|
"rewards/margins": 0.29742380625540205,
|
|
"rewards/rejected": -0.01452856868892521,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 18.890146255493164,
|
|
"kl": 14.805468559265137,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37082898.314785376,
|
|
"logits/rejected": -37032077.56682028,
|
|
"logps/chosen": -495.70160969793324,
|
|
"logps/rejected": -368.2362471198157,
|
|
"loss": 0.5962,
|
|
"loss/base_kto": 3.6946403980255127,
|
|
"metrics/advantage_var": 306.1405334472656,
|
|
"regularization/lipschitz_norm": 1110.30419921875,
|
|
"regularization/w1": 1.0747743844985962,
|
|
"rewards/chosen": 0.2752458241861462,
|
|
"rewards/margins": 0.26083179426635633,
|
|
"rewards/rejected": 0.014414029919789866,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 20.330968856811523,
|
|
"kl": 5.8462653160095215,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35395339.02392344,
|
|
"logits/rejected": -36207197.30474732,
|
|
"logps/chosen": -481.1236044657097,
|
|
"logps/rejected": -363.46815179938744,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.6927356719970703,
|
|
"metrics/advantage_var": 263.9264831542969,
|
|
"regularization/lipschitz_norm": 1075.0716552734375,
|
|
"regularization/w1": 1.040669322013855,
|
|
"rewards/chosen": -0.015467316720284154,
|
|
"rewards/margins": 0.2952021894052901,
|
|
"rewards/rejected": -0.31066950612557426,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 24.9837589263916,
|
|
"kl": 11.190443992614746,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -34349700.20657277,
|
|
"logits/rejected": -36814432.64898596,
|
|
"logps/chosen": -472.0523767605634,
|
|
"logps/rejected": -392.29621684867396,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.7195801734924316,
|
|
"metrics/advantage_var": 276.13665771484375,
|
|
"regularization/lipschitz_norm": 1077.98046875,
|
|
"regularization/w1": 1.0434850454330444,
|
|
"rewards/chosen": 0.1479542333755135,
|
|
"rewards/margins": 0.2727873864916772,
|
|
"rewards/rejected": -0.1248331531161637,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 21.31780242919922,
|
|
"kl": 8.398650169372559,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37238059.8272,
|
|
"logits/rejected": -37278921.673282444,
|
|
"logps/chosen": -469.3322,
|
|
"logps/rejected": -398.4593034351145,
|
|
"loss": 0.5922,
|
|
"loss/base_kto": 3.74658203125,
|
|
"metrics/advantage_var": 238.9662628173828,
|
|
"regularization/lipschitz_norm": 1023.7175903320312,
|
|
"regularization/w1": 0.9909586310386658,
|
|
"rewards/chosen": 0.008236831665039062,
|
|
"rewards/margins": 0.21772090288700946,
|
|
"rewards/rejected": -0.2094840712219704,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 15.567916870117188,
|
|
"kl": 7.363170146942139,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35003254.249594815,
|
|
"logits/rejected": -36386337.2066365,
|
|
"logps/chosen": -494.2009724473258,
|
|
"logps/rejected": -365.56570512820514,
|
|
"loss": 0.5885,
|
|
"loss/base_kto": 3.7361361980438232,
|
|
"metrics/advantage_var": 224.89883422851562,
|
|
"regularization/lipschitz_norm": 1004.3369140625,
|
|
"regularization/w1": 0.9721981287002563,
|
|
"rewards/chosen": 0.06719711649939539,
|
|
"rewards/margins": 0.237335077126104,
|
|
"rewards/rejected": -0.1701379606267086,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 24.032546997070312,
|
|
"kl": 9.157782554626465,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -34908598.409785934,
|
|
"logits/rejected": -36706009.55910543,
|
|
"logps/chosen": -486.85865825688074,
|
|
"logps/rejected": -368.68033646166134,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.7558517456054688,
|
|
"metrics/advantage_var": 231.52017211914062,
|
|
"regularization/lipschitz_norm": 1012.8148803710938,
|
|
"regularization/w1": 0.9804048538208008,
|
|
"rewards/chosen": 0.08251190185546875,
|
|
"rewards/margins": 0.23528511607989716,
|
|
"rewards/rejected": -0.1527732142244284,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 22.605880737304688,
|
|
"kl": 7.248265266418457,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36638179.018867925,
|
|
"logits/rejected": -35327058.68322981,
|
|
"logps/chosen": -476.2138364779874,
|
|
"logps/rejected": -384.40333850931677,
|
|
"loss": 0.5959,
|
|
"loss/base_kto": 3.7408242225646973,
|
|
"metrics/advantage_var": 263.2743835449219,
|
|
"regularization/lipschitz_norm": 1060.4591064453125,
|
|
"regularization/w1": 1.0265244245529175,
|
|
"rewards/chosen": 0.10654787747365124,
|
|
"rewards/margins": 0.25763921023504854,
|
|
"rewards/rejected": -0.15109133276139727,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 27.252864837646484,
|
|
"kl": 11.121315002441406,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35950467.97515528,
|
|
"logits/rejected": -36643794.91823899,
|
|
"logps/chosen": -488.9490489130435,
|
|
"logps/rejected": -365.5615664308176,
|
|
"loss": 0.5998,
|
|
"loss/base_kto": 3.7466704845428467,
|
|
"metrics/advantage_var": 265.4429626464844,
|
|
"regularization/lipschitz_norm": 1086.5301513671875,
|
|
"regularization/w1": 1.0517610311508179,
|
|
"rewards/chosen": 0.152827979615016,
|
|
"rewards/margins": 0.2504669795059972,
|
|
"rewards/rejected": -0.09763899989098122,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 24.75717544555664,
|
|
"kl": 8.752140998840332,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36627251.2,
|
|
"logits/rejected": -36925485.15275591,
|
|
"logps/chosen": -490.5701550387597,
|
|
"logps/rejected": -367.12942913385825,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.7421226501464844,
|
|
"metrics/advantage_var": 273.6818542480469,
|
|
"regularization/lipschitz_norm": 1060.021240234375,
|
|
"regularization/w1": 1.026100516319275,
|
|
"rewards/chosen": 0.15237606551296026,
|
|
"rewards/margins": 0.27113368289269696,
|
|
"rewards/rejected": -0.11875761737973671,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 20.251605987548828,
|
|
"kl": 10.91238784790039,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36909988.71961102,
|
|
"logits/rejected": -37753480.68778281,
|
|
"logps/chosen": -480.24483387358185,
|
|
"logps/rejected": -384.8237415158371,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.7201783657073975,
|
|
"metrics/advantage_var": 244.40771484375,
|
|
"regularization/lipschitz_norm": 1040.3936767578125,
|
|
"regularization/w1": 1.007101058959961,
|
|
"rewards/chosen": 0.1605994473400147,
|
|
"rewards/margins": 0.25788314583481226,
|
|
"rewards/rejected": -0.09728369849479755,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 19.03770637512207,
|
|
"kl": 5.372455596923828,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36305011.9245283,
|
|
"logits/rejected": -36325131.13043478,
|
|
"logps/chosen": -472.3481230345912,
|
|
"logps/rejected": -367.3989955357143,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.7491490840911865,
|
|
"metrics/advantage_var": 242.1458282470703,
|
|
"regularization/lipschitz_norm": 1012.4777221679688,
|
|
"regularization/w1": 0.9800785183906555,
|
|
"rewards/chosen": -0.09079524705994804,
|
|
"rewards/margins": 0.2295552577500772,
|
|
"rewards/rejected": -0.32035050481002525,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 18.146291732788086,
|
|
"kl": 6.706477642059326,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35684746.20150376,
|
|
"logits/rejected": -37200791.10243902,
|
|
"logps/chosen": -505.90216165413534,
|
|
"logps/rejected": -362.4806910569106,
|
|
"loss": 0.5916,
|
|
"loss/base_kto": 3.750601291656494,
|
|
"metrics/advantage_var": 253.0991973876953,
|
|
"regularization/lipschitz_norm": 1014.5658569335938,
|
|
"regularization/w1": 0.9820996522903442,
|
|
"rewards/chosen": 0.04141369583015155,
|
|
"rewards/margins": 0.26068507500634563,
|
|
"rewards/rejected": -0.2192713791761941,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 23.190210342407227,
|
|
"kl": 12.415057182312012,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36821188.55727554,
|
|
"logits/rejected": -37918976.0,
|
|
"logps/chosen": -470.85255417956654,
|
|
"logps/rejected": -362.6338261471519,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.511014938354492,
|
|
"metrics/advantage_var": 425.7567443847656,
|
|
"regularization/lipschitz_norm": 1323.0213623046875,
|
|
"regularization/w1": 1.2806845903396606,
|
|
"rewards/chosen": 0.3479822353681913,
|
|
"rewards/margins": 0.5165544801330746,
|
|
"rewards/rejected": -0.1685722447648833,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 15.094964027404785,
|
|
"kl": 9.411001205444336,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35588362.98349835,
|
|
"logits/rejected": -37161579.869436204,
|
|
"logps/chosen": -472.8260107260726,
|
|
"logps/rejected": -365.7991237017804,
|
|
"loss": 0.6122,
|
|
"loss/base_kto": 3.365062713623047,
|
|
"metrics/advantage_var": 596.436279296875,
|
|
"regularization/lipschitz_norm": 1583.323486328125,
|
|
"regularization/w1": 1.5326570272445679,
|
|
"rewards/chosen": 0.22851139484065594,
|
|
"rewards/margins": 0.6968778875903314,
|
|
"rewards/rejected": -0.46836649274967546,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 23.973220825195312,
|
|
"kl": 10.743574142456055,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -33417089.819905214,
|
|
"logits/rejected": -35526944.04945904,
|
|
"logps/chosen": -465.03297788309635,
|
|
"logps/rejected": -381.89637268160743,
|
|
"loss": 0.6033,
|
|
"loss/base_kto": 3.2513177394866943,
|
|
"metrics/advantage_var": 620.6433715820312,
|
|
"regularization/lipschitz_norm": 1627.0052490234375,
|
|
"regularization/w1": 1.5749410390853882,
|
|
"rewards/chosen": 0.3973120618582148,
|
|
"rewards/margins": 0.856496381237924,
|
|
"rewards/rejected": -0.4591843193797092,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 21.905805587768555,
|
|
"kl": 9.628497123718262,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35349444.92307692,
|
|
"logits/rejected": -37393514.146341465,
|
|
"logps/chosen": -487.56049679487177,
|
|
"logps/rejected": -378.9127286585366,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.323640823364258,
|
|
"metrics/advantage_var": 539.4470825195312,
|
|
"regularization/lipschitz_norm": 1509.3988037109375,
|
|
"regularization/w1": 1.461098074913025,
|
|
"rewards/chosen": 0.36176212017352766,
|
|
"rewards/margins": 0.7473041756291774,
|
|
"rewards/rejected": -0.3855420554556498,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 20.7901668548584,
|
|
"kl": 13.369658470153809,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35224843.76545166,
|
|
"logits/rejected": -36602054.80431433,
|
|
"logps/chosen": -473.06418383518223,
|
|
"logps/rejected": -375.352031972265,
|
|
"loss": 0.6177,
|
|
"loss/base_kto": 3.2955589294433594,
|
|
"metrics/advantage_var": 680.991455078125,
|
|
"regularization/lipschitz_norm": 1700.3582763671875,
|
|
"regularization/w1": 1.6459465026855469,
|
|
"rewards/chosen": 0.3789882266955973,
|
|
"rewards/margins": 0.7962333009986116,
|
|
"rewards/rejected": -0.4172450743030143,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 20.343870162963867,
|
|
"kl": 12.414015769958496,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34090330.73417722,
|
|
"logits/rejected": -35467203.95061728,
|
|
"logps/chosen": -458.1440862341772,
|
|
"logps/rejected": -369.397159529321,
|
|
"loss": 0.6119,
|
|
"loss/base_kto": 3.319322347640991,
|
|
"metrics/advantage_var": 599.7037963867188,
|
|
"regularization/lipschitz_norm": 1628.3851318359375,
|
|
"regularization/w1": 1.5762766599655151,
|
|
"rewards/chosen": 0.3447738840610166,
|
|
"rewards/margins": 0.746129203613968,
|
|
"rewards/rejected": -0.4013553195529514,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 17.768192291259766,
|
|
"kl": 17.229093551635742,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34604595.34670487,
|
|
"logits/rejected": -34350199.64261168,
|
|
"logps/chosen": -463.49086676217763,
|
|
"logps/rejected": -373.56588273195877,
|
|
"loss": 0.5964,
|
|
"loss/base_kto": 3.3150627613067627,
|
|
"metrics/advantage_var": 555.2005004882812,
|
|
"regularization/lipschitz_norm": 1504.4609375,
|
|
"regularization/w1": 1.4563181400299072,
|
|
"rewards/chosen": 0.5407342774137043,
|
|
"rewards/margins": 0.7110109061504821,
|
|
"rewards/rejected": -0.17027662873677782,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 18.17601203918457,
|
|
"kl": 29.794462203979492,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34898743.119617224,
|
|
"logits/rejected": -36604440.30627871,
|
|
"logps/chosen": -466.8919457735247,
|
|
"logps/rejected": -349.51746745788665,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.277207136154175,
|
|
"metrics/advantage_var": 542.3825073242188,
|
|
"regularization/lipschitz_norm": 1537.3563232421875,
|
|
"regularization/w1": 1.4881608486175537,
|
|
"rewards/chosen": 0.7279997951866527,
|
|
"rewards/margins": 0.7635038544451531,
|
|
"rewards/rejected": -0.035504059258500434,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 25.832965850830078,
|
|
"kl": 25.964340209960938,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35796727.94968554,
|
|
"logits/rejected": -37028294.757763974,
|
|
"logps/chosen": -472.3318592767296,
|
|
"logps/rejected": -367.61379076086956,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.2431321144104004,
|
|
"metrics/advantage_var": 626.7291259765625,
|
|
"regularization/lipschitz_norm": 1588.559814453125,
|
|
"regularization/w1": 1.537725806236267,
|
|
"rewards/chosen": 0.6725802151661999,
|
|
"rewards/margins": 0.8015443727445676,
|
|
"rewards/rejected": -0.12896415757836763,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 17.008806228637695,
|
|
"kl": 18.336414337158203,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35578185.19938176,
|
|
"logits/rejected": -36221461.83886256,
|
|
"logps/chosen": -487.70580564142193,
|
|
"logps/rejected": -349.38008491311217,
|
|
"loss": 0.6093,
|
|
"loss/base_kto": 3.2575058937072754,
|
|
"metrics/advantage_var": 690.2684936523438,
|
|
"regularization/lipschitz_norm": 1670.5321044921875,
|
|
"regularization/w1": 1.617074966430664,
|
|
"rewards/chosen": 0.562875550165427,
|
|
"rewards/margins": 0.8442507059834563,
|
|
"rewards/rejected": -0.28137515581802924,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 21.019044876098633,
|
|
"kl": 18.322830200195312,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35494652.545727134,
|
|
"logits/rejected": -35744509.07667211,
|
|
"logps/chosen": -497.87378185907045,
|
|
"logps/rejected": -372.2220126427406,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.2976510524749756,
|
|
"metrics/advantage_var": 664.135498046875,
|
|
"regularization/lipschitz_norm": 1582.7314453125,
|
|
"regularization/w1": 1.5320841073989868,
|
|
"rewards/chosen": 0.602982551082857,
|
|
"rewards/margins": 0.7670870563137591,
|
|
"rewards/rejected": -0.16410450523090206,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 21.914852142333984,
|
|
"kl": 18.818378448486328,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35118670.23298033,
|
|
"logits/rejected": -37378600.52988692,
|
|
"logps/chosen": -499.37414901664147,
|
|
"logps/rejected": -374.53675282714056,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.281919240951538,
|
|
"metrics/advantage_var": 598.3539428710938,
|
|
"regularization/lipschitz_norm": 1589.9345703125,
|
|
"regularization/w1": 1.539056658744812,
|
|
"rewards/chosen": 0.6129649013686649,
|
|
"rewards/margins": 0.7700925528975611,
|
|
"rewards/rejected": -0.15712765152889616,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 19.1385498046875,
|
|
"kl": 18.0656681060791,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34849660.29581994,
|
|
"logits/rejected": -36409113.67781155,
|
|
"logps/chosen": -456.8292805466238,
|
|
"logps/rejected": -376.95618825987845,
|
|
"loss": 0.6008,
|
|
"loss/base_kto": 3.266716480255127,
|
|
"metrics/advantage_var": 567.34765625,
|
|
"regularization/lipschitz_norm": 1590.2733154296875,
|
|
"regularization/w1": 1.5393844842910767,
|
|
"rewards/chosen": 0.5471190424977391,
|
|
"rewards/margins": 0.810753674921395,
|
|
"rewards/rejected": -0.263634632423656,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 20.76739501953125,
|
|
"kl": 12.531976699829102,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35243240.80127186,
|
|
"logits/rejected": -36533523.2688172,
|
|
"logps/chosen": -490.87112480127183,
|
|
"logps/rejected": -383.5096966205837,
|
|
"loss": 0.6112,
|
|
"loss/base_kto": 3.3029744625091553,
|
|
"metrics/advantage_var": 716.6351928710938,
|
|
"regularization/lipschitz_norm": 1639.134033203125,
|
|
"regularization/w1": 1.5866817235946655,
|
|
"rewards/chosen": 0.4233794341216216,
|
|
"rewards/margins": 0.7739105946783973,
|
|
"rewards/rejected": -0.35053116055677563,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 18.617046356201172,
|
|
"kl": 8.331124305725098,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -33056453.670886077,
|
|
"logits/rejected": -35443857.38271605,
|
|
"logps/chosen": -478.7087618670886,
|
|
"logps/rejected": -396.5805362654321,
|
|
"loss": 0.6065,
|
|
"loss/base_kto": 3.3282151222229004,
|
|
"metrics/advantage_var": 571.9895629882812,
|
|
"regularization/lipschitz_norm": 1573.963134765625,
|
|
"regularization/w1": 1.5235962867736816,
|
|
"rewards/chosen": 0.3222663493096074,
|
|
"rewards/margins": 0.7304141847467698,
|
|
"rewards/rejected": -0.40814783543716243,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 15.352473258972168,
|
|
"kl": 11.78324031829834,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35287961.139430285,
|
|
"logits/rejected": -35840016.70473083,
|
|
"logps/chosen": -483.6323088455772,
|
|
"logps/rejected": -375.938927406199,
|
|
"loss": 0.6006,
|
|
"loss/base_kto": 3.31672739982605,
|
|
"metrics/advantage_var": 587.284423828125,
|
|
"regularization/lipschitz_norm": 1537.381103515625,
|
|
"regularization/w1": 1.4881848096847534,
|
|
"rewards/chosen": 0.42149972987139245,
|
|
"rewards/margins": 0.7596054714375884,
|
|
"rewards/rejected": -0.33810574156619594,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 15.512990951538086,
|
|
"kl": 24.919565200805664,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36600577.15837104,
|
|
"logits/rejected": -36697033.23176661,
|
|
"logps/chosen": -462.94994343891403,
|
|
"logps/rejected": -381.0310727309562,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.2668755054473877,
|
|
"metrics/advantage_var": 563.43408203125,
|
|
"regularization/lipschitz_norm": 1583.4862060546875,
|
|
"regularization/w1": 1.5328145027160645,
|
|
"rewards/chosen": 0.6703406056307739,
|
|
"rewards/margins": 0.7802145086796431,
|
|
"rewards/rejected": -0.10987390304886928,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 21.205745697021484,
|
|
"kl": 10.23928165435791,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34873062.839313574,
|
|
"logits/rejected": -35856525.02034429,
|
|
"logps/chosen": -506.283736349454,
|
|
"logps/rejected": -391.4015062597809,
|
|
"loss": 0.6237,
|
|
"loss/base_kto": 3.296105146408081,
|
|
"metrics/advantage_var": 908.1524658203125,
|
|
"regularization/lipschitz_norm": 1749.1607666015625,
|
|
"regularization/w1": 1.6931877136230469,
|
|
"rewards/chosen": 0.37295617923349744,
|
|
"rewards/margins": 0.7983148634568934,
|
|
"rewards/rejected": -0.42535868422339596,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.41119956970215,
|
|
"kl": 13.737591743469238,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -33868203.05631659,
|
|
"logits/rejected": -35198427.0176565,
|
|
"logps/chosen": -511.1039764079148,
|
|
"logps/rejected": -366.57907804975923,
|
|
"loss": 0.6094,
|
|
"loss/base_kto": 3.268270254135132,
|
|
"metrics/advantage_var": 602.8837280273438,
|
|
"regularization/lipschitz_norm": 1659.706298828125,
|
|
"regularization/w1": 1.6065956354141235,
|
|
"rewards/chosen": 0.5461068581591276,
|
|
"rewards/margins": 0.804103467817848,
|
|
"rewards/rejected": -0.2579966096587204,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.09857749938965,
|
|
"kl": 13.377432823181152,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -33628585.23358779,
|
|
"logits/rejected": -36098510.0288,
|
|
"logps/chosen": -457.9335400763359,
|
|
"logps/rejected": -372.3904,
|
|
"loss": 0.6032,
|
|
"loss/base_kto": 3.329882860183716,
|
|
"metrics/advantage_var": 557.4445190429688,
|
|
"regularization/lipschitz_norm": 1544.8406982421875,
|
|
"regularization/w1": 1.4954057931900024,
|
|
"rewards/chosen": 0.47822475287750477,
|
|
"rewards/margins": 0.7075744354946922,
|
|
"rewards/rejected": -0.2293496826171875,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 24.2120361328125,
|
|
"kl": 17.392555236816406,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34461819.74622356,
|
|
"logits/rejected": -35917532.37540453,
|
|
"logps/chosen": -482.06353851963746,
|
|
"logps/rejected": -389.41254550970876,
|
|
"loss": 0.5985,
|
|
"loss/base_kto": 3.3604018688201904,
|
|
"metrics/advantage_var": 578.8988037109375,
|
|
"regularization/lipschitz_norm": 1474.77392578125,
|
|
"regularization/w1": 1.4275811910629272,
|
|
"rewards/chosen": 0.48124007948215636,
|
|
"rewards/margins": 0.6917910728737674,
|
|
"rewards/rejected": -0.21055099339161104,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 20.3862247467041,
|
|
"kl": 22.15376091003418,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35213855.9488,
|
|
"logits/rejected": -35874200.03664122,
|
|
"logps/chosen": -465.654,
|
|
"logps/rejected": -385.3359971374046,
|
|
"loss": 0.617,
|
|
"loss/base_kto": 3.281867742538452,
|
|
"metrics/advantage_var": 818.151123046875,
|
|
"regularization/lipschitz_norm": 1709.1787109375,
|
|
"regularization/w1": 1.6544849872589111,
|
|
"rewards/chosen": 0.568648828125,
|
|
"rewards/margins": 0.7257217777164837,
|
|
"rewards/rejected": -0.15707294959148377,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 15.866477012634277,
|
|
"kl": 16.126449584960938,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -33328958.855284553,
|
|
"logits/rejected": -36385211.47669173,
|
|
"logps/chosen": -500.58958333333334,
|
|
"logps/rejected": -400.2533364661654,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.2381935119628906,
|
|
"metrics/advantage_var": 614.0758666992188,
|
|
"regularization/lipschitz_norm": 1601.5994873046875,
|
|
"regularization/w1": 1.550348162651062,
|
|
"rewards/chosen": 0.5206572741996951,
|
|
"rewards/margins": 0.8061251017886871,
|
|
"rewards/rejected": -0.28546782758899203,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 21.90863037109375,
|
|
"kl": 11.127936363220215,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34104755.77744361,
|
|
"logits/rejected": -34606497.92520325,
|
|
"logps/chosen": -492.30249060150373,
|
|
"logps/rejected": -373.6853150406504,
|
|
"loss": 0.616,
|
|
"loss/base_kto": 3.3320329189300537,
|
|
"metrics/advantage_var": 577.935302734375,
|
|
"regularization/lipschitz_norm": 1649.1051025390625,
|
|
"regularization/w1": 1.5963337421417236,
|
|
"rewards/chosen": 0.4293969175869361,
|
|
"rewards/margins": 0.742410183553336,
|
|
"rewards/rejected": -0.3130132659663999,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.541954040527344,
|
|
"kl": 27.366214752197266,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35602554.55748031,
|
|
"logits/rejected": -37120523.906976745,
|
|
"logps/chosen": -492.4576771653543,
|
|
"logps/rejected": -383.4197189922481,
|
|
"loss": 0.6149,
|
|
"loss/base_kto": 3.2778451442718506,
|
|
"metrics/advantage_var": 654.3119506835938,
|
|
"regularization/lipschitz_norm": 1695.574462890625,
|
|
"regularization/w1": 1.6413160562515259,
|
|
"rewards/chosen": 0.6180968277097687,
|
|
"rewards/margins": 0.7462049602421694,
|
|
"rewards/rejected": -0.12810813253240066,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 21.975324630737305,
|
|
"kl": 14.015891075134277,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35352369.00459418,
|
|
"logits/rejected": -37997004.55502392,
|
|
"logps/chosen": -517.374569295559,
|
|
"logps/rejected": -353.6818181818182,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.2623748779296875,
|
|
"metrics/advantage_var": 588.2733764648438,
|
|
"regularization/lipschitz_norm": 1639.5390625,
|
|
"regularization/w1": 1.5870739221572876,
|
|
"rewards/chosen": 0.5089719342966357,
|
|
"rewards/margins": 0.7935869961767443,
|
|
"rewards/rejected": -0.28461506188010866,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 23.730363845825195,
|
|
"kl": 16.976552963256836,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -35217262.8921095,
|
|
"logits/rejected": -37190623.368740514,
|
|
"logps/chosen": -473.2291666666667,
|
|
"logps/rejected": -365.88569328528075,
|
|
"loss": 0.6008,
|
|
"loss/base_kto": 3.291616201400757,
|
|
"metrics/advantage_var": 571.4057006835938,
|
|
"regularization/lipschitz_norm": 1564.6864013671875,
|
|
"regularization/w1": 1.5146163702011108,
|
|
"rewards/chosen": 0.515448135646261,
|
|
"rewards/margins": 0.7447856050453269,
|
|
"rewards/rejected": -0.22933746939906582,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 16.515743255615234,
|
|
"kl": 15.26294994354248,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34693212.510903426,
|
|
"logits/rejected": -35159454.09404389,
|
|
"logps/chosen": -482.8357671339564,
|
|
"logps/rejected": -382.845954153605,
|
|
"loss": 0.597,
|
|
"loss/base_kto": 3.288755178451538,
|
|
"metrics/advantage_var": 622.9114379882812,
|
|
"regularization/lipschitz_norm": 1536.192626953125,
|
|
"regularization/w1": 1.4870343208312988,
|
|
"rewards/chosen": 0.5087190431969189,
|
|
"rewards/margins": 0.7804539206367944,
|
|
"rewards/rejected": -0.27173487743987557,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 14.575602531433105,
|
|
"kl": 15.25003719329834,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34340322.76672105,
|
|
"logits/rejected": -36499314.75862069,
|
|
"logps/chosen": -470.53125,
|
|
"logps/rejected": -367.58302098950526,
|
|
"loss": 0.6105,
|
|
"loss/base_kto": 3.238189458847046,
|
|
"metrics/advantage_var": 650.7476196289062,
|
|
"regularization/lipschitz_norm": 1700.1590576171875,
|
|
"regularization/w1": 1.6457538604736328,
|
|
"rewards/chosen": 0.508925022542185,
|
|
"rewards/margins": 0.8279714777081566,
|
|
"rewards/rejected": -0.3190464551659717,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.860393524169922,
|
|
"kl": 8.02838134765625,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -33357676.819571864,
|
|
"logits/rejected": -35763837.955271564,
|
|
"logps/chosen": -473.2672974006116,
|
|
"logps/rejected": -363.9890425319489,
|
|
"loss": 0.6002,
|
|
"loss/base_kto": 3.262939453125,
|
|
"metrics/advantage_var": 584.277099609375,
|
|
"regularization/lipschitz_norm": 1589.353271484375,
|
|
"regularization/w1": 1.5384939908981323,
|
|
"rewards/chosen": 0.45235655431718275,
|
|
"rewards/margins": 0.8391085519022965,
|
|
"rewards/rejected": -0.3867519975851138,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 22.32830047607422,
|
|
"kl": 14.150607109069824,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35828974.33065811,
|
|
"logits/rejected": -35446322.654490106,
|
|
"logps/chosen": -500.97602327447834,
|
|
"logps/rejected": -395.9500095129376,
|
|
"loss": 0.6073,
|
|
"loss/base_kto": 3.2730233669281006,
|
|
"metrics/advantage_var": 650.7073364257812,
|
|
"regularization/lipschitz_norm": 1637.4290771484375,
|
|
"regularization/w1": 1.5850313901901245,
|
|
"rewards/chosen": 0.4652767273243128,
|
|
"rewards/margins": 0.8003541147000501,
|
|
"rewards/rejected": -0.3350773873757373,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 16.482961654663086,
|
|
"kl": 9.742484092712402,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34040332.8,
|
|
"logits/rejected": -36287939.2,
|
|
"logps/chosen": -471.75498046875,
|
|
"logps/rejected": -378.1865234375,
|
|
"loss": 0.5993,
|
|
"loss/base_kto": 3.34765887260437,
|
|
"metrics/advantage_var": 576.4119262695312,
|
|
"regularization/lipschitz_norm": 1494.62353515625,
|
|
"regularization/w1": 1.4467957019805908,
|
|
"rewards/chosen": 0.39304141998291015,
|
|
"rewards/margins": 0.7137337684631347,
|
|
"rewards/rejected": -0.3206923484802246,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 28.27539825439453,
|
|
"kl": 13.098017692565918,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -35510078.56071964,
|
|
"logits/rejected": -35925708.63295269,
|
|
"logps/chosen": -470.10625937031483,
|
|
"logps/rejected": -359.27008564437193,
|
|
"loss": 0.6102,
|
|
"loss/base_kto": 3.3460328578948975,
|
|
"metrics/advantage_var": 1050.8026123046875,
|
|
"regularization/lipschitz_norm": 1585.9912109375,
|
|
"regularization/w1": 1.5352394580841064,
|
|
"rewards/chosen": 0.5042811541006841,
|
|
"rewards/margins": 0.7036321303445666,
|
|
"rewards/rejected": -0.19935097624388254,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 20.633995056152344,
|
|
"kl": 14.561180114746094,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34451084.82034976,
|
|
"logits/rejected": -35866457.26574501,
|
|
"logps/chosen": -497.5389507154213,
|
|
"logps/rejected": -386.1611703149002,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.2682251930236816,
|
|
"metrics/advantage_var": 545.0556640625,
|
|
"regularization/lipschitz_norm": 1528.5404052734375,
|
|
"regularization/w1": 1.479627251625061,
|
|
"rewards/chosen": 0.4862970737281399,
|
|
"rewards/margins": 0.7785655601612717,
|
|
"rewards/rejected": -0.29226848643313175,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.701581954956055,
|
|
"kl": 11.452432632446289,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35697291.78159127,
|
|
"logits/rejected": -36275829.78403756,
|
|
"logps/chosen": -480.68125975039004,
|
|
"logps/rejected": -363.0715473395931,
|
|
"loss": 0.6179,
|
|
"loss/base_kto": 3.37455677986145,
|
|
"metrics/advantage_var": 622.552001953125,
|
|
"regularization/lipschitz_norm": 1620.2760009765625,
|
|
"regularization/w1": 1.568427324295044,
|
|
"rewards/chosen": 0.3718554992199688,
|
|
"rewards/margins": 0.6929975137965778,
|
|
"rewards/rejected": -0.32114201457660896,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 14.728906631469727,
|
|
"kl": 16.546998977661133,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34478846.36942675,
|
|
"logits/rejected": -36221625.32515337,
|
|
"logps/chosen": -465.9889032643312,
|
|
"logps/rejected": -390.706384202454,
|
|
"loss": 0.5952,
|
|
"loss/base_kto": 3.328676700592041,
|
|
"metrics/advantage_var": 550.82666015625,
|
|
"regularization/lipschitz_norm": 1480.2578125,
|
|
"regularization/w1": 1.4328895807266235,
|
|
"rewards/chosen": 0.3935989817236639,
|
|
"rewards/margins": 0.689255525291532,
|
|
"rewards/rejected": -0.2956565435678681,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 19.20351219177246,
|
|
"kl": 14.74124813079834,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34962900.06864274,
|
|
"logits/rejected": -37755264.60093897,
|
|
"logps/chosen": -488.10871684867396,
|
|
"logps/rejected": -390.09467918622846,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.341092348098755,
|
|
"metrics/advantage_var": 549.0675659179688,
|
|
"regularization/lipschitz_norm": 1510.6282958984375,
|
|
"regularization/w1": 1.4622882604599,
|
|
"rewards/chosen": 0.4436404604621685,
|
|
"rewards/margins": 0.684837834594113,
|
|
"rewards/rejected": -0.24119737413194445,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 18.116750717163086,
|
|
"kl": 12.19059944152832,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -33270414.08992248,
|
|
"logits/rejected": -35000416.75590551,
|
|
"logps/chosen": -506.17713178294576,
|
|
"logps/rejected": -365.162155511811,
|
|
"loss": 0.6061,
|
|
"loss/base_kto": 3.3585686683654785,
|
|
"metrics/advantage_var": 559.7994995117188,
|
|
"regularization/lipschitz_norm": 1539.2572021484375,
|
|
"regularization/w1": 1.4900009632110596,
|
|
"rewards/chosen": 0.38096188094264777,
|
|
"rewards/margins": 0.7030596592629603,
|
|
"rewards/rejected": -0.3220977783203125,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 14.686551094055176,
|
|
"kl": 16.379161834716797,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33170007.57894737,
|
|
"logits/rejected": -34907986.28571428,
|
|
"logps/chosen": -490.8579358552632,
|
|
"logps/rejected": -368.4364769345238,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.2888245582580566,
|
|
"metrics/advantage_var": 568.0514526367188,
|
|
"regularization/lipschitz_norm": 1554.4371337890625,
|
|
"regularization/w1": 1.504694938659668,
|
|
"rewards/chosen": 0.49511999832956416,
|
|
"rewards/margins": 0.7493636219722586,
|
|
"rewards/rejected": -0.25424362364269437,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 19.832347869873047,
|
|
"kl": 8.920297622680664,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35746486.567353405,
|
|
"logits/rejected": -35967476.12982999,
|
|
"logps/chosen": -500.636043977813,
|
|
"logps/rejected": -373.3530235703246,
|
|
"loss": 0.5697,
|
|
"loss/base_kto": 3.216679096221924,
|
|
"metrics/advantage_var": 485.2485046386719,
|
|
"regularization/lipschitz_norm": 1385.5474853515625,
|
|
"regularization/w1": 1.3412100076675415,
|
|
"rewards/chosen": 0.3797266456857914,
|
|
"rewards/margins": 0.856157830481,
|
|
"rewards/rejected": -0.47643118479520863,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 21.89859962463379,
|
|
"kl": 7.963223934173584,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34025620.35905512,
|
|
"logits/rejected": -35273955.02635659,
|
|
"logps/chosen": -483.9924212598425,
|
|
"logps/rejected": -391.209980620155,
|
|
"loss": 0.5612,
|
|
"loss/base_kto": 3.1962673664093018,
|
|
"metrics/advantage_var": 484.2899475097656,
|
|
"regularization/lipschitz_norm": 1335.9390869140625,
|
|
"regularization/w1": 1.2931890487670898,
|
|
"rewards/chosen": 0.40625403697096457,
|
|
"rewards/margins": 0.9007795146538619,
|
|
"rewards/rejected": -0.4945254776828973,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 19.94541358947754,
|
|
"kl": 10.798842430114746,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33959290.56969697,
|
|
"logits/rejected": -34563745.85806452,
|
|
"logps/chosen": -480.3489583333333,
|
|
"logps/rejected": -382.5594758064516,
|
|
"loss": 0.5601,
|
|
"loss/base_kto": 3.2194650173187256,
|
|
"metrics/advantage_var": 437.97882080078125,
|
|
"regularization/lipschitz_norm": 1303.2080078125,
|
|
"regularization/w1": 1.261505365371704,
|
|
"rewards/chosen": 0.40321266867897726,
|
|
"rewards/margins": 0.8180123158563966,
|
|
"rewards/rejected": -0.41479964717741935,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 15.935685157775879,
|
|
"kl": 13.427490234375,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34735607.76848874,
|
|
"logits/rejected": -36867442.38297872,
|
|
"logps/chosen": -480.03054662379424,
|
|
"logps/rejected": -407.0298252279635,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.1622695922851562,
|
|
"metrics/advantage_var": 454.5888671875,
|
|
"regularization/lipschitz_norm": 1314.315673828125,
|
|
"regularization/w1": 1.272257685661316,
|
|
"rewards/chosen": 0.48861944560452675,
|
|
"rewards/margins": 0.8846590431361092,
|
|
"rewards/rejected": -0.39603959753158247,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 17.077974319458008,
|
|
"kl": 11.499598503112793,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34062842.391236305,
|
|
"logits/rejected": -35439119.176287055,
|
|
"logps/chosen": -455.6285211267606,
|
|
"logps/rejected": -377.0400741029641,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.197096347808838,
|
|
"metrics/advantage_var": 422.5838928222656,
|
|
"regularization/lipschitz_norm": 1278.277587890625,
|
|
"regularization/w1": 1.237372636795044,
|
|
"rewards/chosen": 0.49088799561693075,
|
|
"rewards/margins": 0.8599527075462282,
|
|
"rewards/rejected": -0.36906471192929746,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 15.541845321655273,
|
|
"kl": 7.3202714920043945,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35913826.69879518,
|
|
"logits/rejected": -36105568.415584415,
|
|
"logps/chosen": -477.18844126506025,
|
|
"logps/rejected": -377.83081371753246,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.2012290954589844,
|
|
"metrics/advantage_var": 410.8737487792969,
|
|
"regularization/lipschitz_norm": 1274.033203125,
|
|
"regularization/w1": 1.2332642078399658,
|
|
"rewards/chosen": 0.4088220711213997,
|
|
"rewards/margins": 0.8678787767448778,
|
|
"rewards/rejected": -0.4590567056234781,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 19.03652572631836,
|
|
"kl": 16.03883171081543,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34887383.78849144,
|
|
"logits/rejected": -36789831.535321824,
|
|
"logps/chosen": -498.41757387247276,
|
|
"logps/rejected": -364.00431711145995,
|
|
"loss": 0.5556,
|
|
"loss/base_kto": 3.133333206176758,
|
|
"metrics/advantage_var": 471.56182861328125,
|
|
"regularization/lipschitz_norm": 1355.15771484375,
|
|
"regularization/w1": 1.3117927312850952,
|
|
"rewards/chosen": 0.5923158162118488,
|
|
"rewards/margins": 0.9083339132794017,
|
|
"rewards/rejected": -0.31601809706755296,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 16.680809020996094,
|
|
"kl": 14.453997611999512,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35109820.37735849,
|
|
"logits/rejected": -36396330.93167702,
|
|
"logps/chosen": -477.95735062893084,
|
|
"logps/rejected": -370.841056871118,
|
|
"loss": 0.5515,
|
|
"loss/base_kto": 3.102193593978882,
|
|
"metrics/advantage_var": 468.564453125,
|
|
"regularization/lipschitz_norm": 1352.7255859375,
|
|
"regularization/w1": 1.3094384670257568,
|
|
"rewards/chosen": 0.6048946740492335,
|
|
"rewards/margins": 0.9435668855007935,
|
|
"rewards/rejected": -0.33867221145156007,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 24.052738189697266,
|
|
"kl": 12.60477352142334,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34349540.36825397,
|
|
"logits/rejected": -34623465.94461539,
|
|
"logps/chosen": -466.6015873015873,
|
|
"logps/rejected": -383.36846153846153,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.164311170578003,
|
|
"metrics/advantage_var": 476.12921142578125,
|
|
"regularization/lipschitz_norm": 1392.194580078125,
|
|
"regularization/w1": 1.3476442098617554,
|
|
"rewards/chosen": 0.5004607669890873,
|
|
"rewards/margins": 0.896481697352669,
|
|
"rewards/rejected": -0.3960209303635817,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 15.353602409362793,
|
|
"kl": 9.640913963317871,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35780358.56410257,
|
|
"logits/rejected": -37144394.92682927,
|
|
"logps/chosen": -477.7608173076923,
|
|
"logps/rejected": -368.784775152439,
|
|
"loss": 0.5489,
|
|
"loss/base_kto": 3.1865601539611816,
|
|
"metrics/advantage_var": 413.68359375,
|
|
"regularization/lipschitz_norm": 1244.8675537109375,
|
|
"regularization/w1": 1.2050317525863647,
|
|
"rewards/chosen": 0.45113671131623095,
|
|
"rewards/margins": 0.8710790089028115,
|
|
"rewards/rejected": -0.4199422975865806,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 19.645570755004883,
|
|
"kl": 11.93309211730957,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34557834.44108761,
|
|
"logits/rejected": -35913317.074433655,
|
|
"logps/chosen": -487.8172205438066,
|
|
"logps/rejected": -369.0038683252427,
|
|
"loss": 0.557,
|
|
"loss/base_kto": 3.16609787940979,
|
|
"metrics/advantage_var": 439.7220153808594,
|
|
"regularization/lipschitz_norm": 1332.4222412109375,
|
|
"regularization/w1": 1.2897846698760986,
|
|
"rewards/chosen": 0.48893438149074775,
|
|
"rewards/margins": 0.8944033191392864,
|
|
"rewards/rejected": -0.40546893764853864,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 13.989500999450684,
|
|
"kl": 14.450139045715332,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34658866.06940063,
|
|
"logits/rejected": -35601557.00309598,
|
|
"logps/chosen": -485.98077681388014,
|
|
"logps/rejected": -389.7189193111455,
|
|
"loss": 0.5648,
|
|
"loss/base_kto": 3.164170980453491,
|
|
"metrics/advantage_var": 473.74200439453125,
|
|
"regularization/lipschitz_norm": 1398.80517578125,
|
|
"regularization/w1": 1.3540433645248413,
|
|
"rewards/chosen": 0.5909651987936219,
|
|
"rewards/margins": 0.8866910182898002,
|
|
"rewards/rejected": -0.29572581949617843,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 16.189327239990234,
|
|
"kl": 11.753397941589355,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34559513.12883436,
|
|
"logits/rejected": -36000852.789808914,
|
|
"logps/chosen": -465.643884202454,
|
|
"logps/rejected": -370.72123805732485,
|
|
"loss": 0.5641,
|
|
"loss/base_kto": 3.1884517669677734,
|
|
"metrics/advantage_var": 502.8970642089844,
|
|
"regularization/lipschitz_norm": 1368.1595458984375,
|
|
"regularization/w1": 1.3243783712387085,
|
|
"rewards/chosen": 0.5534203652223927,
|
|
"rewards/margins": 0.8631156637270703,
|
|
"rewards/rejected": -0.30969529850467753,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 20.154483795166016,
|
|
"kl": 11.416178703308105,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35384447.424287856,
|
|
"logits/rejected": -36804126.90375204,
|
|
"logps/chosen": -525.9418103448276,
|
|
"logps/rejected": -358.78206566068513,
|
|
"loss": 0.5615,
|
|
"loss/base_kto": 3.1734509468078613,
|
|
"metrics/advantage_var": 462.114013671875,
|
|
"regularization/lipschitz_norm": 1361.947265625,
|
|
"regularization/w1": 1.3183649778366089,
|
|
"rewards/chosen": 0.5241920665643741,
|
|
"rewards/margins": 0.8779498408302795,
|
|
"rewards/rejected": -0.3537577742659054,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 16.525434494018555,
|
|
"kl": 12.25172233581543,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34165764.777604975,
|
|
"logits/rejected": -36022619.22762951,
|
|
"logps/chosen": -491.60604587869364,
|
|
"logps/rejected": -367.30298273155415,
|
|
"loss": 0.5709,
|
|
"loss/base_kto": 3.1733152866363525,
|
|
"metrics/advantage_var": 476.2198791503906,
|
|
"regularization/lipschitz_norm": 1439.91650390625,
|
|
"regularization/w1": 1.3938392400741577,
|
|
"rewards/chosen": 0.5377036846629568,
|
|
"rewards/margins": 0.8708096133325016,
|
|
"rewards/rejected": -0.33310592866954475,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 15.079082489013672,
|
|
"kl": 12.411714553833008,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34833135.2599681,
|
|
"logits/rejected": -35935838.87289433,
|
|
"logps/chosen": -451.6981658692185,
|
|
"logps/rejected": -386.6646248085758,
|
|
"loss": 0.561,
|
|
"loss/base_kto": 3.231250047683716,
|
|
"metrics/advantage_var": 405.32867431640625,
|
|
"regularization/lipschitz_norm": 1298.2681884765625,
|
|
"regularization/w1": 1.2567236423492432,
|
|
"rewards/chosen": 0.43814432487913674,
|
|
"rewards/margins": 0.7966030965190438,
|
|
"rewards/rejected": -0.35845877163990714,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 14.774227142333984,
|
|
"kl": 12.383523941040039,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33579212.8,
|
|
"logits/rejected": -36042826.76825397,
|
|
"logps/chosen": -476.85942307692306,
|
|
"logps/rejected": -371.9820188492063,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.1879355907440186,
|
|
"metrics/advantage_var": 465.83770751953125,
|
|
"regularization/lipschitz_norm": 1356.3656005859375,
|
|
"regularization/w1": 1.3129619359970093,
|
|
"rewards/chosen": 0.46731121356670674,
|
|
"rewards/margins": 0.8773125505272722,
|
|
"rewards/rejected": -0.41000133696056545,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 17.516733169555664,
|
|
"kl": 11.263381004333496,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34234809.2631579,
|
|
"logits/rejected": -36706703.23809524,
|
|
"logps/chosen": -476.1243832236842,
|
|
"logps/rejected": -377.2783435639881,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.1911723613739014,
|
|
"metrics/advantage_var": 454.2433166503906,
|
|
"regularization/lipschitz_norm": 1328.9354248046875,
|
|
"regularization/w1": 1.2864094972610474,
|
|
"rewards/chosen": 0.46600713227924545,
|
|
"rewards/margins": 0.8416058580976978,
|
|
"rewards/rejected": -0.3755987258184524,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 16.33338165283203,
|
|
"kl": 10.109529495239258,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34452697.403076924,
|
|
"logits/rejected": -36515004.54603174,
|
|
"logps/chosen": -478.72778846153847,
|
|
"logps/rejected": -361.4546130952381,
|
|
"loss": 0.5563,
|
|
"loss/base_kto": 3.1940488815307617,
|
|
"metrics/advantage_var": 450.54400634765625,
|
|
"regularization/lipschitz_norm": 1298.1900634765625,
|
|
"regularization/w1": 1.256648063659668,
|
|
"rewards/chosen": 0.5028693096454326,
|
|
"rewards/margins": 0.8506774451473548,
|
|
"rewards/rejected": -0.34780813550192213,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 17.103260040283203,
|
|
"kl": 11.305916786193848,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34056343.58425197,
|
|
"logits/rejected": -35604441.89767442,
|
|
"logps/chosen": -510.2220964566929,
|
|
"logps/rejected": -384.35329457364344,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.210218906402588,
|
|
"metrics/advantage_var": 398.03277587890625,
|
|
"regularization/lipschitz_norm": 1289.62451171875,
|
|
"regularization/w1": 1.2483564615249634,
|
|
"rewards/chosen": 0.4129806878998524,
|
|
"rewards/margins": 0.8291276567709764,
|
|
"rewards/rejected": -0.416146968871124,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 13.693824768066406,
|
|
"kl": 11.231901168823242,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34975075.5328,
|
|
"logits/rejected": -36562037.2519084,
|
|
"logps/chosen": -461.2134,
|
|
"logps/rejected": -379.83284828244274,
|
|
"loss": 0.5532,
|
|
"loss/base_kto": 3.1471829414367676,
|
|
"metrics/advantage_var": 453.8047790527344,
|
|
"regularization/lipschitz_norm": 1320.3148193359375,
|
|
"regularization/w1": 1.2780648469924927,
|
|
"rewards/chosen": 0.476273095703125,
|
|
"rewards/margins": 0.8842373915344703,
|
|
"rewards/rejected": -0.4079642958313454,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 21.8228702545166,
|
|
"kl": 10.097284317016602,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35680366.276923075,
|
|
"logits/rejected": -36090766.222222224,
|
|
"logps/chosen": -485.4498076923077,
|
|
"logps/rejected": -395.991691468254,
|
|
"loss": 0.5596,
|
|
"loss/base_kto": 3.2041866779327393,
|
|
"metrics/advantage_var": 447.53546142578125,
|
|
"regularization/lipschitz_norm": 1314.6365966796875,
|
|
"regularization/w1": 1.2725682258605957,
|
|
"rewards/chosen": 0.42717961237980767,
|
|
"rewards/margins": 0.8594246791890834,
|
|
"rewards/rejected": -0.43224506680927577,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 19.715198516845703,
|
|
"kl": 12.224166870117188,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34828281.2852459,
|
|
"logits/rejected": -36145191.737313434,
|
|
"logps/chosen": -475.6319159836066,
|
|
"logps/rejected": -402.1400186567164,
|
|
"loss": 0.5627,
|
|
"loss/base_kto": 3.2128219604492188,
|
|
"metrics/advantage_var": 442.60357666015625,
|
|
"regularization/lipschitz_norm": 1331.3394775390625,
|
|
"regularization/w1": 1.2887365818023682,
|
|
"rewards/chosen": 0.46946871397925205,
|
|
"rewards/margins": 0.8268080548724424,
|
|
"rewards/rejected": -0.3573393408931903,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 17.652387619018555,
|
|
"kl": 11.485856056213379,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35123821.88871224,
|
|
"logits/rejected": -35522978.40860215,
|
|
"logps/chosen": -507.59807233704294,
|
|
"logps/rejected": -361.008784562212,
|
|
"loss": 0.5576,
|
|
"loss/base_kto": 3.13618803024292,
|
|
"metrics/advantage_var": 487.19732666015625,
|
|
"regularization/lipschitz_norm": 1368.5726318359375,
|
|
"regularization/w1": 1.324778437614441,
|
|
"rewards/chosen": 0.49694688369435613,
|
|
"rewards/margins": 0.9343460491159431,
|
|
"rewards/rejected": -0.43739916542158697,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 20.829431533813477,
|
|
"kl": 9.707969665527344,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35104294.4,
|
|
"logits/rejected": -35396448.0,
|
|
"logps/chosen": -499.63681640625,
|
|
"logps/rejected": -408.16787109375,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.158587694168091,
|
|
"metrics/advantage_var": 461.05224609375,
|
|
"regularization/lipschitz_norm": 1362.4744873046875,
|
|
"regularization/w1": 1.3188751935958862,
|
|
"rewards/chosen": 0.4626021385192871,
|
|
"rewards/margins": 0.8942239284515381,
|
|
"rewards/rejected": -0.431621789932251,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 25.18584442138672,
|
|
"kl": 10.333015441894531,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34953715.2,
|
|
"logits/rejected": -36164294.4,
|
|
"logps/chosen": -499.62099609375,
|
|
"logps/rejected": -378.3279296875,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.1900126934051514,
|
|
"metrics/advantage_var": 433.4665222167969,
|
|
"regularization/lipschitz_norm": 1356.5330810546875,
|
|
"regularization/w1": 1.3131239414215088,
|
|
"rewards/chosen": 0.4079261779785156,
|
|
"rewards/margins": 0.8461101531982422,
|
|
"rewards/rejected": -0.4381839752197266,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 16.694236755371094,
|
|
"kl": 9.096259117126465,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -33919099.3010279,
|
|
"logits/rejected": -36184412.74123539,
|
|
"logps/chosen": -504.5181718061674,
|
|
"logps/rejected": -352.5863418196995,
|
|
"loss": 0.5737,
|
|
"loss/base_kto": 3.1659293174743652,
|
|
"metrics/advantage_var": 824.1036987304688,
|
|
"regularization/lipschitz_norm": 1470.7811279296875,
|
|
"regularization/w1": 1.4237160682678223,
|
|
"rewards/chosen": 0.4760711714734765,
|
|
"rewards/margins": 0.8588093135347896,
|
|
"rewards/rejected": -0.3827381420613131,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 17.056594848632812,
|
|
"kl": 9.983824729919434,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -32986793.589905363,
|
|
"logits/rejected": -36054085.746130034,
|
|
"logps/chosen": -475.39910291798105,
|
|
"logps/rejected": -352.9758126934984,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.222651720046997,
|
|
"metrics/advantage_var": 444.0147399902344,
|
|
"regularization/lipschitz_norm": 1331.890625,
|
|
"regularization/w1": 1.2892701625823975,
|
|
"rewards/chosen": 0.4675080212132788,
|
|
"rewards/margins": 0.823879874217947,
|
|
"rewards/rejected": -0.35637185300466817,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 12.820006370544434,
|
|
"kl": 11.673685073852539,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34266621.588697016,
|
|
"logits/rejected": -36960013.934681185,
|
|
"logps/chosen": -493.0527864992151,
|
|
"logps/rejected": -376.27007192846037,
|
|
"loss": 0.5469,
|
|
"loss/base_kto": 3.1460659503936768,
|
|
"metrics/advantage_var": 466.6276550292969,
|
|
"regularization/lipschitz_norm": 1269.818603515625,
|
|
"regularization/w1": 1.2291842699050903,
|
|
"rewards/chosen": 0.515916761461195,
|
|
"rewards/margins": 0.9079495567992102,
|
|
"rewards/rejected": -0.39203279533801516,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.636578559875488,
|
|
"kl": 11.184309959411621,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35773777.13230769,
|
|
"logits/rejected": -37253471.08571429,
|
|
"logps/chosen": -485.21740384615384,
|
|
"logps/rejected": -381.7684027777778,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.125328540802002,
|
|
"metrics/advantage_var": 481.3175964355469,
|
|
"regularization/lipschitz_norm": 1379.0472412109375,
|
|
"regularization/w1": 1.3349177837371826,
|
|
"rewards/chosen": 0.5428059269831731,
|
|
"rewards/margins": 0.9327230644924737,
|
|
"rewards/rejected": -0.38991713750930057,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 18.770830154418945,
|
|
"kl": 11.703639030456543,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34847018.12658228,
|
|
"logits/rejected": -35032209.38271605,
|
|
"logps/chosen": -483.4439280063291,
|
|
"logps/rejected": -353.645881558642,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.1920933723449707,
|
|
"metrics/advantage_var": 452.3001403808594,
|
|
"regularization/lipschitz_norm": 1301.8756103515625,
|
|
"regularization/w1": 1.2602156400680542,
|
|
"rewards/chosen": 0.5000883174847953,
|
|
"rewards/margins": 0.867645866667075,
|
|
"rewards/rejected": -0.3675575491822796,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 15.435226440429688,
|
|
"kl": 13.41156005859375,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35750206.35897436,
|
|
"logits/rejected": -35239427.12195122,
|
|
"logps/chosen": -460.6829427083333,
|
|
"logps/rejected": -404.6037538109756,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.1568715572357178,
|
|
"metrics/advantage_var": 443.94873046875,
|
|
"regularization/lipschitz_norm": 1336.811767578125,
|
|
"regularization/w1": 1.2940337657928467,
|
|
"rewards/chosen": 0.4726253900772486,
|
|
"rewards/margins": 0.8782601538414206,
|
|
"rewards/rejected": -0.40563476376417207,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 20.130613327026367,
|
|
"kl": 9.79339599609375,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34129891.15492958,
|
|
"logits/rejected": -35186700.780031204,
|
|
"logps/chosen": -465.1637323943662,
|
|
"logps/rejected": -375.63772425897037,
|
|
"loss": 0.5465,
|
|
"loss/base_kto": 3.1563546657562256,
|
|
"metrics/advantage_var": 434.60137939453125,
|
|
"regularization/lipschitz_norm": 1255.7060546875,
|
|
"regularization/w1": 1.2155234813690186,
|
|
"rewards/chosen": 0.5008374902190923,
|
|
"rewards/margins": 0.8796890371032333,
|
|
"rewards/rejected": -0.37885154688414097,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 17.81679916381836,
|
|
"kl": 10.139800071716309,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35804687.900621116,
|
|
"logits/rejected": -36437314.012578614,
|
|
"logps/chosen": -496.7468944099379,
|
|
"logps/rejected": -370.1814809355346,
|
|
"loss": 0.5637,
|
|
"loss/base_kto": 3.197054624557495,
|
|
"metrics/advantage_var": 472.7958984375,
|
|
"regularization/lipschitz_norm": 1356.276611328125,
|
|
"regularization/w1": 1.312875747680664,
|
|
"rewards/chosen": 0.5197515073029891,
|
|
"rewards/margins": 0.87249687534882,
|
|
"rewards/rejected": -0.3527453680458309,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 12.215582847595215,
|
|
"kl": 12.430460929870605,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35006370.6163522,
|
|
"logits/rejected": -36758954.136645965,
|
|
"logps/chosen": -470.7425314465409,
|
|
"logps/rejected": -372.70060170807454,
|
|
"loss": 0.5562,
|
|
"loss/base_kto": 3.1164519786834717,
|
|
"metrics/advantage_var": 489.5458679199219,
|
|
"regularization/lipschitz_norm": 1377.111083984375,
|
|
"regularization/w1": 1.3330434560775757,
|
|
"rewards/chosen": 0.5485211258414406,
|
|
"rewards/margins": 0.9483621501099286,
|
|
"rewards/rejected": -0.39984102426848794,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 16.069026947021484,
|
|
"kl": 10.979535102844238,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35466033.951219514,
|
|
"logits/rejected": -35640411.897435896,
|
|
"logps/chosen": -470.0374428353659,
|
|
"logps/rejected": -390.1776091746795,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.1961257457733154,
|
|
"metrics/advantage_var": 467.57598876953125,
|
|
"regularization/lipschitz_norm": 1361.852294921875,
|
|
"regularization/w1": 1.3182729482650757,
|
|
"rewards/chosen": 0.4945542870498285,
|
|
"rewards/margins": 0.8762905533571703,
|
|
"rewards/rejected": -0.38173626630734175,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 14.50257396697998,
|
|
"kl": 10.80820369720459,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34907956.17061611,
|
|
"logits/rejected": -36849914.064914994,
|
|
"logps/chosen": -478.53929699842024,
|
|
"logps/rejected": -363.3722952086553,
|
|
"loss": 0.5602,
|
|
"loss/base_kto": 3.1777961254119873,
|
|
"metrics/advantage_var": 454.50732421875,
|
|
"regularization/lipschitz_norm": 1346.4949951171875,
|
|
"regularization/w1": 1.303407073020935,
|
|
"rewards/chosen": 0.4962760840923677,
|
|
"rewards/margins": 0.8795880912830072,
|
|
"rewards/rejected": -0.3833120071906395,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 16.197853088378906,
|
|
"kl": 12.25694751739502,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33690625.60250391,
|
|
"logits/rejected": -35501629.503900155,
|
|
"logps/chosen": -475.36551251956183,
|
|
"logps/rejected": -351.76489372074883,
|
|
"loss": 0.5538,
|
|
"loss/base_kto": 3.1732850074768066,
|
|
"metrics/advantage_var": 491.0556640625,
|
|
"regularization/lipschitz_norm": 1298.767578125,
|
|
"regularization/w1": 1.2572071552276611,
|
|
"rewards/chosen": 0.5119681918005429,
|
|
"rewards/margins": 0.8778844751803937,
|
|
"rewards/rejected": -0.3659162833798508,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_loss": 0.5863274930878213,
|
|
"train_runtime": 11040.2984,
|
|
"train_samples_per_second": 13.425,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.972016217523487e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|