2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 19.502885818481445,
|
|
"kl": 10.540353775024414,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36465607.55905512,
|
|
"logits/rejected": -37725719.02015504,
|
|
"logps/chosen": -481.9946358267716,
|
|
"logps/rejected": -355.0954941860465,
|
|
"loss": 0.6159,
|
|
"loss/base_kto": 3.9690608978271484,
|
|
"metrics/advantage_var": 284.63482666015625,
|
|
"regularization/lipschitz_norm": 989.6254272460938,
|
|
"regularization/w1": 0.9579572677612305,
|
|
"rewards/chosen": 0.051199436938668796,
|
|
"rewards/margins": 0.025122905713272116,
|
|
"rewards/rejected": 0.02607653122539668,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 27.192712783813477,
|
|
"kl": 9.43039608001709,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35914288.0,
|
|
"logits/rejected": -36640915.2,
|
|
"logps/chosen": -475.236328125,
|
|
"logps/rejected": -411.322314453125,
|
|
"loss": 0.5993,
|
|
"loss/base_kto": 3.9007058143615723,
|
|
"metrics/advantage_var": 186.2182159423828,
|
|
"regularization/lipschitz_norm": 923.6155395507812,
|
|
"regularization/w1": 0.8940597772598267,
|
|
"rewards/chosen": 0.12400479316711426,
|
|
"rewards/margins": 0.09270426332950592,
|
|
"rewards/rejected": 0.03130052983760834,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 21.91861915588379,
|
|
"kl": 3.3268611431121826,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35618500.42767295,
|
|
"logits/rejected": -38025820.22360248,
|
|
"logps/chosen": -488.8591784591195,
|
|
"logps/rejected": -374.319099378882,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.904193162918091,
|
|
"metrics/advantage_var": 170.676513671875,
|
|
"regularization/lipschitz_norm": 856.4830322265625,
|
|
"regularization/w1": 0.8290756344795227,
|
|
"rewards/chosen": -0.0567803532822327,
|
|
"rewards/margins": 0.09270326228499241,
|
|
"rewards/rejected": -0.14948361556722511,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 24.039026260375977,
|
|
"kl": 7.054551601409912,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37571532.47798742,
|
|
"logits/rejected": -37501557.664596274,
|
|
"logps/chosen": -505.03439465408803,
|
|
"logps/rejected": -384.27651882763973,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.8435561656951904,
|
|
"metrics/advantage_var": 189.51492309570312,
|
|
"regularization/lipschitz_norm": 917.7451171875,
|
|
"regularization/w1": 0.8883771896362305,
|
|
"rewards/chosen": 0.06644154794561039,
|
|
"rewards/margins": 0.13446009988463287,
|
|
"rewards/rejected": -0.06801855193902247,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 20.555822372436523,
|
|
"kl": 7.473544597625732,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36567167.6012461,
|
|
"logits/rejected": -37179090.25705329,
|
|
"logps/chosen": -508.49805295950154,
|
|
"logps/rejected": -379.4511412617555,
|
|
"loss": 0.6026,
|
|
"loss/base_kto": 3.854013204574585,
|
|
"metrics/advantage_var": 207.60972595214844,
|
|
"regularization/lipschitz_norm": 998.3677978515625,
|
|
"regularization/w1": 0.9664199948310852,
|
|
"rewards/chosen": 0.09264791494589357,
|
|
"rewards/margins": 0.13374322406887004,
|
|
"rewards/rejected": -0.041095309122976466,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 24.31682586669922,
|
|
"kl": 11.020273208618164,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36103840.642967544,
|
|
"logits/rejected": -37562423.00157978,
|
|
"logps/chosen": -468.54907264296753,
|
|
"logps/rejected": -369.5820003949447,
|
|
"loss": 0.5966,
|
|
"loss/base_kto": 3.8179566860198975,
|
|
"metrics/advantage_var": 212.6905975341797,
|
|
"regularization/lipschitz_norm": 986.4797973632812,
|
|
"regularization/w1": 0.9549123644828796,
|
|
"rewards/chosen": 0.1772851133272858,
|
|
"rewards/margins": 0.16353663553010148,
|
|
"rewards/rejected": 0.013748477797184322,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 22.040069580078125,
|
|
"kl": 8.687570571899414,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36815411.66972477,
|
|
"logits/rejected": -38144729.55910543,
|
|
"logps/chosen": -480.0016724006116,
|
|
"logps/rejected": -384.2175768769968,
|
|
"loss": 0.5982,
|
|
"loss/base_kto": 3.877420425415039,
|
|
"metrics/advantage_var": 202.13455200195312,
|
|
"regularization/lipschitz_norm": 938.0945434570312,
|
|
"regularization/w1": 0.9080755114555359,
|
|
"rewards/chosen": 0.1087964842443437,
|
|
"rewards/margins": 0.09742651320362408,
|
|
"rewards/rejected": 0.011369971040719615,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 20.883745193481445,
|
|
"kl": 10.727806091308594,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36511668.38153846,
|
|
"logits/rejected": -37984633.092063494,
|
|
"logps/chosen": -493.43442307692305,
|
|
"logps/rejected": -360.281001984127,
|
|
"loss": 0.5934,
|
|
"loss/base_kto": 3.813587188720703,
|
|
"metrics/advantage_var": 217.04295349121094,
|
|
"regularization/lipschitz_norm": 964.0695190429688,
|
|
"regularization/w1": 0.9332193732261658,
|
|
"rewards/chosen": 0.1932104961688702,
|
|
"rewards/margins": 0.18949819173568336,
|
|
"rewards/rejected": 0.003712304433186849,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 23.493505477905273,
|
|
"kl": 14.198290824890137,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35914897.37285491,
|
|
"logits/rejected": -36928481.55242567,
|
|
"logps/chosen": -525.1220261310452,
|
|
"logps/rejected": -365.8267556729264,
|
|
"loss": 0.5964,
|
|
"loss/base_kto": 3.7751405239105225,
|
|
"metrics/advantage_var": 233.9683837890625,
|
|
"regularization/lipschitz_norm": 1028.890625,
|
|
"regularization/w1": 0.9959660768508911,
|
|
"rewards/chosen": 0.32154477219127825,
|
|
"rewards/margins": 0.19677216157914626,
|
|
"rewards/rejected": 0.12477261061213199,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 20.15189552307129,
|
|
"kl": 11.43596363067627,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -35893670.5007874,
|
|
"logits/rejected": -37364194.62945736,
|
|
"logps/chosen": -461.1228346456693,
|
|
"logps/rejected": -373.52916666666664,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.791618824005127,
|
|
"metrics/advantage_var": 250.49156188964844,
|
|
"regularization/lipschitz_norm": 1008.9990234375,
|
|
"regularization/w1": 0.976711094379425,
|
|
"rewards/chosen": 0.2018682524913878,
|
|
"rewards/margins": 0.19489063743065543,
|
|
"rewards/rejected": 0.006977615060732346,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 18.255399703979492,
|
|
"kl": 8.833267211914062,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37039876.03785489,
|
|
"logits/rejected": -38297929.708978325,
|
|
"logps/chosen": -476.2287066246057,
|
|
"logps/rejected": -378.4968798374613,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.823476791381836,
|
|
"metrics/advantage_var": 226.752685546875,
|
|
"regularization/lipschitz_norm": 1011.2144775390625,
|
|
"regularization/w1": 0.978855550289154,
|
|
"rewards/chosen": 0.16825854665473433,
|
|
"rewards/margins": 0.1719990658476691,
|
|
"rewards/rejected": -0.0037405191929347744,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 24.5849552154541,
|
|
"kl": 5.663272857666016,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -34663893.95169082,
|
|
"logits/rejected": -37238451.471927166,
|
|
"logps/chosen": -482.56516706924316,
|
|
"logps/rejected": -350.7662414643399,
|
|
"loss": 0.5842,
|
|
"loss/base_kto": 3.7473397254943848,
|
|
"metrics/advantage_var": 205.72341918945312,
|
|
"regularization/lipschitz_norm": 956.6349487304688,
|
|
"regularization/w1": 0.9260226488113403,
|
|
"rewards/chosen": 0.07158149805237897,
|
|
"rewards/margins": 0.21758951634921841,
|
|
"rewards/rejected": -0.14600801829683943,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 24.188934326171875,
|
|
"kl": 5.694093227386475,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37298240.60567823,
|
|
"logits/rejected": -37633775.356037155,
|
|
"logps/chosen": -484.80047318611986,
|
|
"logps/rejected": -389.95225425696594,
|
|
"loss": 0.6016,
|
|
"loss/base_kto": 3.8440544605255127,
|
|
"metrics/advantage_var": 227.355712890625,
|
|
"regularization/lipschitz_norm": 1000.8671875,
|
|
"regularization/w1": 0.9688394665718079,
|
|
"rewards/chosen": 0.007320325833389812,
|
|
"rewards/margins": 0.13643714003632737,
|
|
"rewards/rejected": -0.12911681420293755,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 23.15621566772461,
|
|
"kl": 7.4383392333984375,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37166114.44648318,
|
|
"logits/rejected": -38387947.55271565,
|
|
"logps/chosen": -494.1877866972477,
|
|
"logps/rejected": -377.49608126996804,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.752016544342041,
|
|
"metrics/advantage_var": 247.40090942382812,
|
|
"regularization/lipschitz_norm": 1035.0667724609375,
|
|
"regularization/w1": 1.001944661140442,
|
|
"rewards/chosen": 0.10213843762692326,
|
|
"rewards/margins": 0.26397890693852466,
|
|
"rewards/rejected": -0.16184046931160143,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 21.181142807006836,
|
|
"kl": 9.91641616821289,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35469505.3312,
|
|
"logits/rejected": -36555249.1480916,
|
|
"logps/chosen": -496.0407,
|
|
"logps/rejected": -396.88645038167937,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.8104965686798096,
|
|
"metrics/advantage_var": 224.2691192626953,
|
|
"regularization/lipschitz_norm": 957.1937866210938,
|
|
"regularization/w1": 0.926563560962677,
|
|
"rewards/chosen": 0.1381375,
|
|
"rewards/margins": 0.16297421930910067,
|
|
"rewards/rejected": -0.02483671930910067,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 28.925504684448242,
|
|
"kl": 5.655398368835449,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35968009.630094044,
|
|
"logits/rejected": -37111980.26168224,
|
|
"logps/chosen": -472.4402919278997,
|
|
"logps/rejected": -366.23666277258565,
|
|
"loss": 0.6015,
|
|
"loss/base_kto": 3.7756805419921875,
|
|
"metrics/advantage_var": 253.16775512695312,
|
|
"regularization/lipschitz_norm": 1070.8963623046875,
|
|
"regularization/w1": 1.0366276502609253,
|
|
"rewards/chosen": 0.021652853974728003,
|
|
"rewards/margins": 0.2172105403734332,
|
|
"rewards/rejected": -0.19555768639870522,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 19.699970245361328,
|
|
"kl": 8.982659339904785,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -37057390.58893871,
|
|
"logits/rejected": -37999698.12111293,
|
|
"logps/chosen": -457.1128082959641,
|
|
"logps/rejected": -360.17824263502456,
|
|
"loss": 0.5889,
|
|
"loss/base_kto": 3.7216057777404785,
|
|
"metrics/advantage_var": 232.3740997314453,
|
|
"regularization/lipschitz_norm": 1022.27587890625,
|
|
"regularization/w1": 0.98956298828125,
|
|
"rewards/chosen": 0.1241162675022188,
|
|
"rewards/margins": 0.27249517974585113,
|
|
"rewards/rejected": -0.14837891224363237,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 18.612668991088867,
|
|
"kl": 16.07741928100586,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36435666.05128205,
|
|
"logits/rejected": -36860150.63414634,
|
|
"logps/chosen": -484.2530548878205,
|
|
"logps/rejected": -373.5055259146341,
|
|
"loss": 0.5924,
|
|
"loss/base_kto": 3.777944326400757,
|
|
"metrics/advantage_var": 228.4033966064453,
|
|
"regularization/lipschitz_norm": 993.0189819335938,
|
|
"regularization/w1": 0.9612423181533813,
|
|
"rewards/chosen": 0.20697818658290765,
|
|
"rewards/margins": 0.18308485497378646,
|
|
"rewards/rejected": 0.023893331609121184,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 20.576034545898438,
|
|
"kl": 11.32768440246582,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -38164690.131699845,
|
|
"logits/rejected": -37973215.74481659,
|
|
"logps/chosen": -482.7297090352221,
|
|
"logps/rejected": -372.47398325358853,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.77555513381958,
|
|
"metrics/advantage_var": 259.54144287109375,
|
|
"regularization/lipschitz_norm": 1034.5167236328125,
|
|
"regularization/w1": 1.0014121532440186,
|
|
"rewards/chosen": 0.16667953421109302,
|
|
"rewards/margins": 0.21787361111422177,
|
|
"rewards/rejected": -0.05119407690312874,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 25.6004581451416,
|
|
"kl": 8.384028434753418,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -37435408.81444992,
|
|
"logits/rejected": -39214450.837555885,
|
|
"logps/chosen": -491.5823583743842,
|
|
"logps/rejected": -375.3518535767511,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.750048875808716,
|
|
"metrics/advantage_var": 260.51397705078125,
|
|
"regularization/lipschitz_norm": 1067.34765625,
|
|
"regularization/w1": 1.03319251537323,
|
|
"rewards/chosen": 0.12343069994195145,
|
|
"rewards/margins": 0.22820004708590635,
|
|
"rewards/rejected": -0.10476934714395492,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 18.828384399414062,
|
|
"kl": 6.43539571762085,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36946130.77726574,
|
|
"logits/rejected": -37258896.0763116,
|
|
"logps/chosen": -490.66013824884794,
|
|
"logps/rejected": -368.1537162162162,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.710017442703247,
|
|
"metrics/advantage_var": 257.0143737792969,
|
|
"regularization/lipschitz_norm": 1050.6287841796875,
|
|
"regularization/w1": 1.017008662223816,
|
|
"rewards/chosen": 0.10299192742085493,
|
|
"rewards/margins": 0.2900332419143466,
|
|
"rewards/rejected": -0.18704131449349165,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 24.25418472290039,
|
|
"kl": 7.056011199951172,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35502141.5015015,
|
|
"logits/rejected": -38496542.8534202,
|
|
"logps/chosen": -509.06109234234236,
|
|
"logps/rejected": -362.90182206840393,
|
|
"loss": 0.5885,
|
|
"loss/base_kto": 3.7959816455841064,
|
|
"metrics/advantage_var": 222.783203125,
|
|
"regularization/lipschitz_norm": 942.2185668945312,
|
|
"regularization/w1": 0.9120674133300781,
|
|
"rewards/chosen": 0.020014339023166232,
|
|
"rewards/margins": 0.22314458267869097,
|
|
"rewards/rejected": -0.20313024365552473,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 22.75732421875,
|
|
"kl": 13.806530952453613,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35155249.05636071,
|
|
"logits/rejected": -36857427.13201821,
|
|
"logps/chosen": -472.3225644122383,
|
|
"logps/rejected": -352.860252276176,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.754936933517456,
|
|
"metrics/advantage_var": 263.02349853515625,
|
|
"regularization/lipschitz_norm": 1046.362548828125,
|
|
"regularization/w1": 1.0128788948059082,
|
|
"rewards/chosen": 0.25149671275066676,
|
|
"rewards/margins": 0.22947709202025654,
|
|
"rewards/rejected": 0.022019620730410215,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.754810333251953,
|
|
"kl": 9.927902221679688,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36945498.546623796,
|
|
"logits/rejected": -37727879.392097265,
|
|
"logps/chosen": -483.88951969453376,
|
|
"logps/rejected": -387.713359612462,
|
|
"loss": 0.5866,
|
|
"loss/base_kto": 3.695801258087158,
|
|
"metrics/advantage_var": 257.5388488769531,
|
|
"regularization/lipschitz_norm": 1029.981201171875,
|
|
"regularization/w1": 0.9970216751098633,
|
|
"rewards/chosen": 0.12242858739527859,
|
|
"rewards/margins": 0.28709992003576706,
|
|
"rewards/rejected": -0.16467133264048847,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 22.165098190307617,
|
|
"kl": 4.308849811553955,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35867977.708978325,
|
|
"logits/rejected": -37421288.58044164,
|
|
"logps/chosen": -499.8220297987616,
|
|
"logps/rejected": -357.11211060725554,
|
|
"loss": 0.606,
|
|
"loss/base_kto": 3.779498338699341,
|
|
"metrics/advantage_var": 277.275390625,
|
|
"regularization/lipschitz_norm": 1103.4239501953125,
|
|
"regularization/w1": 1.0681142807006836,
|
|
"rewards/chosen": -0.04204266728262414,
|
|
"rewards/margins": 0.22179440697309355,
|
|
"rewards/rejected": -0.2638370742557177,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 13.865706443786621,
|
|
"kl": 12.955889701843262,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35983135.95069338,
|
|
"logits/rejected": -36354843.182250395,
|
|
"logps/chosen": -485.92849576271186,
|
|
"logps/rejected": -367.1637777337559,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.706853151321411,
|
|
"metrics/advantage_var": 266.7944030761719,
|
|
"regularization/lipschitz_norm": 1048.5203857421875,
|
|
"regularization/w1": 1.014967679977417,
|
|
"rewards/chosen": 0.1546203190079088,
|
|
"rewards/margins": 0.23570909696413836,
|
|
"rewards/rejected": -0.08108877795622957,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 23.74933624267578,
|
|
"kl": 17.0903377532959,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37003612.530120485,
|
|
"logits/rejected": -37028145.87012987,
|
|
"logps/chosen": -480.86779932228916,
|
|
"logps/rejected": -357.82533482142856,
|
|
"loss": 0.5865,
|
|
"loss/base_kto": 3.7183785438537598,
|
|
"metrics/advantage_var": 245.51023864746094,
|
|
"regularization/lipschitz_norm": 1006.0230712890625,
|
|
"regularization/w1": 0.9738303422927856,
|
|
"rewards/chosen": 0.25568573733410205,
|
|
"rewards/margins": 0.2609121071361559,
|
|
"rewards/rejected": -0.005226369802053873,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 21.68670082092285,
|
|
"kl": 16.158279418945312,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35993225.36585366,
|
|
"logits/rejected": -37798738.05128205,
|
|
"logps/chosen": -466.3261242378049,
|
|
"logps/rejected": -347.64393028846155,
|
|
"loss": 0.5929,
|
|
"loss/base_kto": 3.717130422592163,
|
|
"metrics/advantage_var": 267.6878356933594,
|
|
"regularization/lipschitz_norm": 1059.6279296875,
|
|
"regularization/w1": 1.0257198810577393,
|
|
"rewards/chosen": 0.27874783771794015,
|
|
"rewards/margins": 0.24696195833827644,
|
|
"rewards/rejected": 0.03178587937966371,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.29503059387207,
|
|
"kl": 19.612791061401367,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37443186.44705883,
|
|
"logits/rejected": -39133450.24,
|
|
"logps/chosen": -473.89393382352944,
|
|
"logps/rejected": -384.88861979166666,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.7605836391448975,
|
|
"metrics/advantage_var": 264.0092468261719,
|
|
"regularization/lipschitz_norm": 1057.7274169921875,
|
|
"regularization/w1": 1.0238802433013916,
|
|
"rewards/chosen": 0.3193834192612592,
|
|
"rewards/margins": 0.18395683438170188,
|
|
"rewards/rejected": 0.1354265848795573,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 19.369218826293945,
|
|
"kl": 23.78046989440918,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37131724.01230769,
|
|
"logits/rejected": -36776235.073015876,
|
|
"logps/chosen": -448.91375,
|
|
"logps/rejected": -393.12395833333335,
|
|
"loss": 0.5839,
|
|
"loss/base_kto": 3.705796957015991,
|
|
"metrics/advantage_var": 228.5234832763672,
|
|
"regularization/lipschitz_norm": 997.0167846679688,
|
|
"regularization/w1": 0.9651121497154236,
|
|
"rewards/chosen": 0.3852544227013221,
|
|
"rewards/margins": 0.24248262007157878,
|
|
"rewards/rejected": 0.14277180262974332,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 17.708091735839844,
|
|
"kl": 28.061298370361328,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35577591.94968554,
|
|
"logits/rejected": -37040554.136645965,
|
|
"logps/chosen": -484.1072621855346,
|
|
"logps/rejected": -386.75875873447205,
|
|
"loss": 0.591,
|
|
"loss/base_kto": 3.7084076404571533,
|
|
"metrics/advantage_var": 254.99778747558594,
|
|
"regularization/lipschitz_norm": 1053.327392578125,
|
|
"regularization/w1": 1.0196208953857422,
|
|
"rewards/chosen": 0.4374661235689367,
|
|
"rewards/margins": 0.26452037832588793,
|
|
"rewards/rejected": 0.17294574524304882,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 19.906936645507812,
|
|
"kl": 24.466257095336914,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35220891.91599354,
|
|
"logits/rejected": -36624117.54311649,
|
|
"logps/chosen": -490.69870759289176,
|
|
"logps/rejected": -365.4009786308623,
|
|
"loss": 0.5907,
|
|
"loss/base_kto": 3.6738502979278564,
|
|
"metrics/advantage_var": 248.9242401123047,
|
|
"regularization/lipschitz_norm": 1086.1605224609375,
|
|
"regularization/w1": 1.0514034032821655,
|
|
"rewards/chosen": 0.39608597023614195,
|
|
"rewards/margins": 0.29161436247963546,
|
|
"rewards/rejected": 0.10447160775650648,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 26.147384643554688,
|
|
"kl": 12.144790649414062,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36715623.33944954,
|
|
"logits/rejected": -38034494.15974441,
|
|
"logps/chosen": -495.59193425076455,
|
|
"logps/rejected": -371.1836561501597,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.681264877319336,
|
|
"metrics/advantage_var": 258.9990234375,
|
|
"regularization/lipschitz_norm": 1108.2406005859375,
|
|
"regularization/w1": 1.0727767944335938,
|
|
"rewards/chosen": 0.18698968216549,
|
|
"rewards/margins": 0.2901096511526106,
|
|
"rewards/rejected": -0.10311996898712061,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 20.63196563720703,
|
|
"kl": 14.935086250305176,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35298580.8896,
|
|
"logits/rejected": -37464167.18167939,
|
|
"logps/chosen": -473.2944,
|
|
"logps/rejected": -375.1194895038168,
|
|
"loss": 0.5962,
|
|
"loss/base_kto": 3.7402260303497314,
|
|
"metrics/advantage_var": 265.4290466308594,
|
|
"regularization/lipschitz_norm": 1063.7518310546875,
|
|
"regularization/w1": 1.0297117233276367,
|
|
"rewards/chosen": 0.1980023193359375,
|
|
"rewards/margins": 0.21393468492813694,
|
|
"rewards/rejected": -0.01593236559219943,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 19.969995498657227,
|
|
"kl": 11.184234619140625,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35064625.8913738,
|
|
"logits/rejected": -37724122.42201835,
|
|
"logps/chosen": -484.41363817891374,
|
|
"logps/rejected": -362.4001099006116,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.7191121578216553,
|
|
"metrics/advantage_var": 280.61505126953125,
|
|
"regularization/lipschitz_norm": 1108.084228515625,
|
|
"regularization/w1": 1.0726255178451538,
|
|
"rewards/chosen": 0.16599346806827825,
|
|
"rewards/margins": 0.25389966683234433,
|
|
"rewards/rejected": -0.08790619876406608,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 22.121328353881836,
|
|
"kl": 2.6912906169891357,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36159004.39936609,
|
|
"logits/rejected": -37235132.942989215,
|
|
"logps/chosen": -485.33572702060223,
|
|
"logps/rejected": -372.68104776579355,
|
|
"loss": 0.604,
|
|
"loss/base_kto": 3.73968768119812,
|
|
"metrics/advantage_var": 286.8072204589844,
|
|
"regularization/lipschitz_norm": 1128.5030517578125,
|
|
"regularization/w1": 1.0923908948898315,
|
|
"rewards/chosen": -0.08005631895715197,
|
|
"rewards/margins": 0.2564960467617425,
|
|
"rewards/rejected": -0.33655236571889446,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 27.893657684326172,
|
|
"kl": 10.677139282226562,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -37224357.83647799,
|
|
"logits/rejected": -38481156.770186335,
|
|
"logps/chosen": -516.0352790880503,
|
|
"logps/rejected": -354.36563470496895,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.6745967864990234,
|
|
"metrics/advantage_var": 281.8489685058594,
|
|
"regularization/lipschitz_norm": 1124.1219482421875,
|
|
"regularization/w1": 1.0881500244140625,
|
|
"rewards/chosen": 0.13159493980167797,
|
|
"rewards/margins": 0.2987112352785559,
|
|
"rewards/rejected": -0.16711629547687792,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 25.167612075805664,
|
|
"kl": 8.667101860046387,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -37289690.02870814,
|
|
"logits/rejected": -36736741.73353752,
|
|
"logps/chosen": -490.4902312599681,
|
|
"logps/rejected": -398.20032542113324,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.738227128982544,
|
|
"metrics/advantage_var": 238.026123046875,
|
|
"regularization/lipschitz_norm": 1030.64697265625,
|
|
"regularization/w1": 0.9976662993431091,
|
|
"rewards/chosen": 0.0029461433252459318,
|
|
"rewards/margins": 0.21809474196553594,
|
|
"rewards/rejected": -0.21514859864029,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 23.108564376831055,
|
|
"kl": 4.641408443450928,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36553831.975384615,
|
|
"logits/rejected": -36972270.06369427,
|
|
"logps/chosen": -481.7581730769231,
|
|
"logps/rejected": -400.83407145700636,
|
|
"loss": 0.601,
|
|
"loss/base_kto": 3.568951368331909,
|
|
"metrics/advantage_var": 399.6943054199219,
|
|
"regularization/lipschitz_norm": 1279.939697265625,
|
|
"regularization/w1": 1.2389816045761108,
|
|
"rewards/chosen": 0.0772690171461839,
|
|
"rewards/margins": 0.4604759617029365,
|
|
"rewards/rejected": -0.3832069445567526,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 21.707332611083984,
|
|
"kl": 14.657580375671387,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36976424.421052635,
|
|
"logits/rejected": -36639666.47318612,
|
|
"logps/chosen": -466.5954431114551,
|
|
"logps/rejected": -384.6578765772871,
|
|
"loss": 0.6076,
|
|
"loss/base_kto": 3.317751407623291,
|
|
"metrics/advantage_var": 683.1608276367188,
|
|
"regularization/lipschitz_norm": 1594.18115234375,
|
|
"regularization/w1": 1.5431673526763916,
|
|
"rewards/chosen": 0.45354903371710525,
|
|
"rewards/margins": 0.7501314452334573,
|
|
"rewards/rejected": -0.296582411516352,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 23.63801383972168,
|
|
"kl": 6.508891582489014,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35412135.86885246,
|
|
"logits/rejected": -37316852.53731343,
|
|
"logps/chosen": -492.6726434426229,
|
|
"logps/rejected": -370.5968516791045,
|
|
"loss": 0.6261,
|
|
"loss/base_kto": 3.3825745582580566,
|
|
"metrics/advantage_var": 714.6371459960938,
|
|
"regularization/lipschitz_norm": 1679.685546875,
|
|
"regularization/w1": 1.6259357929229736,
|
|
"rewards/chosen": 0.1970718008572938,
|
|
"rewards/margins": 0.6835363895349991,
|
|
"rewards/rejected": -0.4864645886777052,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 20.250513076782227,
|
|
"kl": 4.688612461090088,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36091351.98724083,
|
|
"logits/rejected": -37121282.74425727,
|
|
"logps/chosen": -471.32201953748006,
|
|
"logps/rejected": -356.77323411179174,
|
|
"loss": 0.6109,
|
|
"loss/base_kto": 3.324321746826172,
|
|
"metrics/advantage_var": 593.8258666992188,
|
|
"regularization/lipschitz_norm": 1614.431396484375,
|
|
"regularization/w1": 1.5627692937850952,
|
|
"rewards/chosen": 0.21553630920118122,
|
|
"rewards/margins": 0.7408601507109198,
|
|
"rewards/rejected": -0.5253238415097387,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 20.18074607849121,
|
|
"kl": 3.811614751815796,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -33763671.535483874,
|
|
"logits/rejected": -35160566.69090909,
|
|
"logps/chosen": -497.00176411290323,
|
|
"logps/rejected": -371.0525804924242,
|
|
"loss": 0.6074,
|
|
"loss/base_kto": 3.319126605987549,
|
|
"metrics/advantage_var": 549.409423828125,
|
|
"regularization/lipschitz_norm": 1590.9503173828125,
|
|
"regularization/w1": 1.5400398969650269,
|
|
"rewards/chosen": 0.14914430187594507,
|
|
"rewards/margins": 0.7504365363545199,
|
|
"rewards/rejected": -0.6012922344785748,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 20.280155181884766,
|
|
"kl": 10.081683158874512,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34919102.25981873,
|
|
"logits/rejected": -36171103.275080904,
|
|
"logps/chosen": -492.0630664652568,
|
|
"logps/rejected": -385.86685881877025,
|
|
"loss": 0.6115,
|
|
"loss/base_kto": 3.277081251144409,
|
|
"metrics/advantage_var": 603.0822143554688,
|
|
"regularization/lipschitz_norm": 1668.4169921875,
|
|
"regularization/w1": 1.6150274276733398,
|
|
"rewards/chosen": 0.4720064376416163,
|
|
"rewards/margins": 0.8032982536370148,
|
|
"rewards/rejected": -0.3312918159953985,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.93511390686035,
|
|
"kl": 17.762853622436523,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35067382.518518515,
|
|
"logits/rejected": -36220014.17721519,
|
|
"logps/chosen": -472.04345100308643,
|
|
"logps/rejected": -394.5238825158228,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.2093875408172607,
|
|
"metrics/advantage_var": 677.7565307617188,
|
|
"regularization/lipschitz_norm": 1668.6796875,
|
|
"regularization/w1": 1.6152817010879517,
|
|
"rewards/chosen": 0.576112441074701,
|
|
"rewards/margins": 0.8810555257914979,
|
|
"rewards/rejected": -0.3049430847167969,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 22.243013381958008,
|
|
"kl": 13.155038833618164,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -34833513.93103448,
|
|
"logits/rejected": -36335510.728971966,
|
|
"logps/chosen": -472.52742946708463,
|
|
"logps/rejected": -406.8421923676012,
|
|
"loss": 0.6111,
|
|
"loss/base_kto": 3.2806904315948486,
|
|
"metrics/advantage_var": 660.793701171875,
|
|
"regularization/lipschitz_norm": 1660.9359130859375,
|
|
"regularization/w1": 1.6077858209609985,
|
|
"rewards/chosen": 0.5550422788040018,
|
|
"rewards/margins": 0.8309172158674388,
|
|
"rewards/rejected": -0.275874937063437,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 19.57620620727539,
|
|
"kl": 13.517132759094238,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34961283.2,
|
|
"logits/rejected": -36704915.2,
|
|
"logps/chosen": -470.84365234375,
|
|
"logps/rejected": -386.0208251953125,
|
|
"loss": 0.6085,
|
|
"loss/base_kto": 3.2586090564727783,
|
|
"metrics/advantage_var": 684.2760620117188,
|
|
"regularization/lipschitz_norm": 1662.1953125,
|
|
"regularization/w1": 1.6090049743652344,
|
|
"rewards/chosen": 0.49140443801879885,
|
|
"rewards/margins": 0.859717869758606,
|
|
"rewards/rejected": -0.36831343173980713,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 22.200786590576172,
|
|
"kl": 10.088123321533203,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35913521.93188854,
|
|
"logits/rejected": -36683772.76971609,
|
|
"logps/chosen": -480.27297794117646,
|
|
"logps/rejected": -369.08009660883283,
|
|
"loss": 0.6025,
|
|
"loss/base_kto": 3.2865891456604004,
|
|
"metrics/advantage_var": 568.7406616210938,
|
|
"regularization/lipschitz_norm": 1584.318115234375,
|
|
"regularization/w1": 1.5336198806762695,
|
|
"rewards/chosen": 0.4096281919686049,
|
|
"rewards/margins": 0.7864768640592497,
|
|
"rewards/rejected": -0.3768486720906447,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 15.048163414001465,
|
|
"kl": 14.884773254394531,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -36902889.52978057,
|
|
"logits/rejected": -36496470.13084112,
|
|
"logps/chosen": -472.3496277429467,
|
|
"logps/rejected": -373.2016647196262,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.2245941162109375,
|
|
"metrics/advantage_var": 663.2969970703125,
|
|
"regularization/lipschitz_norm": 1616.8109130859375,
|
|
"regularization/w1": 1.565073013305664,
|
|
"rewards/chosen": 0.5358344290323765,
|
|
"rewards/margins": 0.8439339796539084,
|
|
"rewards/rejected": -0.3080995506215318,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 21.05152130126953,
|
|
"kl": 11.399931907653809,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35074743.19266055,
|
|
"logits/rejected": -35102962.095846646,
|
|
"logps/chosen": -482.59040519877675,
|
|
"logps/rejected": -364.06941393769966,
|
|
"loss": 0.6071,
|
|
"loss/base_kto": 3.337589979171753,
|
|
"metrics/advantage_var": 598.3036499023438,
|
|
"regularization/lipschitz_norm": 1569.380126953125,
|
|
"regularization/w1": 1.5191600322723389,
|
|
"rewards/chosen": 0.4118689674120795,
|
|
"rewards/margins": 0.7199828699336699,
|
|
"rewards/rejected": -0.30811390252159043,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 21.505075454711914,
|
|
"kl": 14.496016502380371,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -36287763.05501618,
|
|
"logits/rejected": -38413463.58912387,
|
|
"logps/chosen": -487.09607605177996,
|
|
"logps/rejected": -363.65955438066464,
|
|
"loss": 0.6221,
|
|
"loss/base_kto": 3.310117483139038,
|
|
"metrics/advantage_var": 672.169921875,
|
|
"regularization/lipschitz_norm": 1721.9129638671875,
|
|
"regularization/w1": 1.6668118238449097,
|
|
"rewards/chosen": 0.4549153645833333,
|
|
"rewards/margins": 0.7606811446605732,
|
|
"rewards/rejected": -0.3057657800772399,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 20.21479606628418,
|
|
"kl": 6.966130256652832,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35867506.75862069,
|
|
"logits/rejected": -37409772.85981309,
|
|
"logps/chosen": -482.1465027429467,
|
|
"logps/rejected": -397.14057632398755,
|
|
"loss": 0.6106,
|
|
"loss/base_kto": 3.3363540172576904,
|
|
"metrics/advantage_var": 611.5675048828125,
|
|
"regularization/lipschitz_norm": 1599.962158203125,
|
|
"regularization/w1": 1.5487632751464844,
|
|
"rewards/chosen": 0.2815138957343505,
|
|
"rewards/margins": 0.7264348960872515,
|
|
"rewards/rejected": -0.4449210003529011,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 18.598018646240234,
|
|
"kl": 7.499533176422119,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -36838896.53418124,
|
|
"logits/rejected": -38404561.59754224,
|
|
"logps/chosen": -476.47595389507154,
|
|
"logps/rejected": -375.0410906298003,
|
|
"loss": 0.6114,
|
|
"loss/base_kto": 3.3419594764709473,
|
|
"metrics/advantage_var": 604.3067016601562,
|
|
"regularization/lipschitz_norm": 1600.061767578125,
|
|
"regularization/w1": 1.54885995388031,
|
|
"rewards/chosen": 0.3158003620578299,
|
|
"rewards/margins": 0.7237542836088091,
|
|
"rewards/rejected": -0.40795392155097926,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 18.696746826171875,
|
|
"kl": 10.276098251342773,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35871946.27160494,
|
|
"logits/rejected": -36755238.88607595,
|
|
"logps/chosen": -493.4796971450617,
|
|
"logps/rejected": -367.6726661392405,
|
|
"loss": 0.6059,
|
|
"loss/base_kto": 3.296455144882202,
|
|
"metrics/advantage_var": 641.5988159179688,
|
|
"regularization/lipschitz_norm": 1602.1904296875,
|
|
"regularization/w1": 1.5509203672409058,
|
|
"rewards/chosen": 0.3852063873667776,
|
|
"rewards/margins": 0.7766391132972933,
|
|
"rewards/rejected": -0.3914327259305157,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 23.424943923950195,
|
|
"kl": 17.93558692932129,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35412436.81927711,
|
|
"logits/rejected": -36924243.116883114,
|
|
"logps/chosen": -475.80600527108436,
|
|
"logps/rejected": -362.1549310064935,
|
|
"loss": 0.5938,
|
|
"loss/base_kto": 3.2207086086273193,
|
|
"metrics/advantage_var": 603.4947509765625,
|
|
"regularization/lipschitz_norm": 1580.6385498046875,
|
|
"regularization/w1": 1.5300581455230713,
|
|
"rewards/chosen": 0.5849044983645519,
|
|
"rewards/margins": 0.8120840380326475,
|
|
"rewards/rejected": -0.22717953966809556,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 20.06446075439453,
|
|
"kl": 18.270017623901367,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34846242.13333333,
|
|
"logits/rejected": -36456448.0,
|
|
"logps/chosen": -472.7660037878788,
|
|
"logps/rejected": -385.2599798387097,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.301849365234375,
|
|
"metrics/advantage_var": 574.2578125,
|
|
"regularization/lipschitz_norm": 1550.585205078125,
|
|
"regularization/w1": 1.5009663105010986,
|
|
"rewards/chosen": 0.6028447931463068,
|
|
"rewards/margins": 0.7754056757146661,
|
|
"rewards/rejected": -0.17256088256835939,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 22.687108993530273,
|
|
"kl": 14.024619102478027,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -35260870.37942122,
|
|
"logits/rejected": -36402695.78115501,
|
|
"logps/chosen": -493.31656953376205,
|
|
"logps/rejected": -358.5651358282675,
|
|
"loss": 0.5941,
|
|
"loss/base_kto": 3.271535634994507,
|
|
"metrics/advantage_var": 542.3544311523438,
|
|
"regularization/lipschitz_norm": 1530.0885009765625,
|
|
"regularization/w1": 1.4811257123947144,
|
|
"rewards/chosen": 0.4445283159949005,
|
|
"rewards/margins": 0.7700361322052444,
|
|
"rewards/rejected": -0.32550781621034386,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 17.669137954711914,
|
|
"kl": 8.48005485534668,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35689878.66028708,
|
|
"logits/rejected": -37940545.47013783,
|
|
"logps/chosen": -492.61996610845296,
|
|
"logps/rejected": -385.4135719754977,
|
|
"loss": 0.6103,
|
|
"loss/base_kto": 3.272345781326294,
|
|
"metrics/advantage_var": 634.9539184570312,
|
|
"regularization/lipschitz_norm": 1662.95703125,
|
|
"regularization/w1": 1.6097424030303955,
|
|
"rewards/chosen": 0.4063806366502193,
|
|
"rewards/margins": 0.8396401420579529,
|
|
"rewards/rejected": -0.43325950540773356,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 18.764556884765625,
|
|
"kl": 7.655716896057129,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35324816.29090909,
|
|
"logits/rejected": -37945350.606451616,
|
|
"logps/chosen": -497.80359848484846,
|
|
"logps/rejected": -380.22545362903224,
|
|
"loss": 0.6118,
|
|
"loss/base_kto": 3.3245739936828613,
|
|
"metrics/advantage_var": 657.45263671875,
|
|
"regularization/lipschitz_norm": 1621.3114013671875,
|
|
"regularization/w1": 1.5694293975830078,
|
|
"rewards/chosen": 0.3568672411369555,
|
|
"rewards/margins": 0.7656576080173109,
|
|
"rewards/rejected": -0.40879036688035536,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 15.40014362335205,
|
|
"kl": 11.943018913269043,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34674464.10289389,
|
|
"logits/rejected": -37781715.64741641,
|
|
"logps/chosen": -484.85595860128615,
|
|
"logps/rejected": -360.4221599544073,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.3243699073791504,
|
|
"metrics/advantage_var": 562.2562866210938,
|
|
"regularization/lipschitz_norm": 1514.376953125,
|
|
"regularization/w1": 1.4659168720245361,
|
|
"rewards/chosen": 0.37278548850890525,
|
|
"rewards/margins": 0.7420156255541654,
|
|
"rewards/rejected": -0.36923013704526025,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 22.178861618041992,
|
|
"kl": 13.211504936218262,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35604978.33594976,
|
|
"logits/rejected": -36639974.91757387,
|
|
"logps/chosen": -458.63196624803766,
|
|
"logps/rejected": -372.6860662908243,
|
|
"loss": 0.6009,
|
|
"loss/base_kto": 3.3307721614837646,
|
|
"metrics/advantage_var": 545.2909545898438,
|
|
"regularization/lipschitz_norm": 1525.1048583984375,
|
|
"regularization/w1": 1.4763014316558838,
|
|
"rewards/chosen": 0.41560490726488913,
|
|
"rewards/margins": 0.7154599036806162,
|
|
"rewards/rejected": -0.29985499641572705,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 17.139915466308594,
|
|
"kl": 21.02091407775879,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35640636.16589862,
|
|
"logits/rejected": -36109824.81399046,
|
|
"logps/chosen": -483.10407066052227,
|
|
"logps/rejected": -373.8740063593005,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.269247055053711,
|
|
"metrics/advantage_var": 638.0188598632812,
|
|
"regularization/lipschitz_norm": 1632.2216796875,
|
|
"regularization/w1": 1.5799906253814697,
|
|
"rewards/chosen": 0.6041948872227823,
|
|
"rewards/margins": 0.7513199372929581,
|
|
"rewards/rejected": -0.14712505007017587,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 22.58076286315918,
|
|
"kl": 12.304976463317871,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -37145650.30744849,
|
|
"logits/rejected": -37580184.65331279,
|
|
"logps/chosen": -485.2071117274168,
|
|
"logps/rejected": -381.2500481510015,
|
|
"loss": 0.6064,
|
|
"loss/base_kto": 3.312950849533081,
|
|
"metrics/advantage_var": 587.2614135742188,
|
|
"regularization/lipschitz_norm": 1589.1822509765625,
|
|
"regularization/w1": 1.5383285284042358,
|
|
"rewards/chosen": 0.3874983266114055,
|
|
"rewards/margins": 0.7483143299729473,
|
|
"rewards/rejected": -0.3608160033615418,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 20.456029891967773,
|
|
"kl": 18.623559951782227,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -36132550.4,
|
|
"logits/rejected": -37032246.4,
|
|
"logps/chosen": -488.74267578125,
|
|
"logps/rejected": -378.44921875,
|
|
"loss": 0.6098,
|
|
"loss/base_kto": 3.263530731201172,
|
|
"metrics/advantage_var": 672.0540161132812,
|
|
"regularization/lipschitz_norm": 1668.3641357421875,
|
|
"regularization/w1": 1.6149762868881226,
|
|
"rewards/chosen": 0.6179819583892823,
|
|
"rewards/margins": 0.7957575678825379,
|
|
"rewards/rejected": -0.1777756094932556,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 39.04401779174805,
|
|
"kl": 12.975259780883789,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -36409393.2012012,
|
|
"logits/rejected": -37002957.13355049,
|
|
"logps/chosen": -469.85003753753756,
|
|
"logps/rejected": -373.6398615635179,
|
|
"loss": 0.6097,
|
|
"loss/base_kto": 3.346444845199585,
|
|
"metrics/advantage_var": 609.91796875,
|
|
"regularization/lipschitz_norm": 1581.406494140625,
|
|
"regularization/w1": 1.5308014154434204,
|
|
"rewards/chosen": 0.3746953969961172,
|
|
"rewards/margins": 0.7253256694793215,
|
|
"rewards/rejected": -0.3506302724832044,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 16.829036712646484,
|
|
"kl": 14.289198875427246,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -36603518.79623824,
|
|
"logits/rejected": -38857973.632398754,
|
|
"logps/chosen": -481.2250685736677,
|
|
"logps/rejected": -380.7901820482866,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.2686493396759033,
|
|
"metrics/advantage_var": 619.9890747070312,
|
|
"regularization/lipschitz_norm": 1619.6253662109375,
|
|
"regularization/w1": 1.5677971839904785,
|
|
"rewards/chosen": 0.5292085749228547,
|
|
"rewards/margins": 0.7895484485249404,
|
|
"rewards/rejected": -0.26033987360208577,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 21.601669311523438,
|
|
"kl": 17.686233520507812,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34984169.516178735,
|
|
"logits/rejected": -37288770.12995245,
|
|
"logps/chosen": -481.24393297380584,
|
|
"logps/rejected": -358.0900356576862,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.270494222640991,
|
|
"metrics/advantage_var": 539.9309692382812,
|
|
"regularization/lipschitz_norm": 1568.1802978515625,
|
|
"regularization/w1": 1.5179985761642456,
|
|
"rewards/chosen": 0.5457217270861422,
|
|
"rewards/margins": 0.7565945646424763,
|
|
"rewards/rejected": -0.21087283755633418,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 23.597270965576172,
|
|
"kl": 18.187515258789062,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35832218.88401254,
|
|
"logits/rejected": -35849914.61682243,
|
|
"logps/chosen": -469.2854134012539,
|
|
"logps/rejected": -364.78353777258565,
|
|
"loss": 0.5906,
|
|
"loss/base_kto": 3.319166898727417,
|
|
"metrics/advantage_var": 526.5990600585938,
|
|
"regularization/lipschitz_norm": 1451.9844970703125,
|
|
"regularization/w1": 1.4055209159851074,
|
|
"rewards/chosen": 0.5910874130583855,
|
|
"rewards/margins": 0.716573709715104,
|
|
"rewards/rejected": -0.1254862966567185,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 20.447795867919922,
|
|
"kl": 10.97195816040039,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -36258186.995137766,
|
|
"logits/rejected": -37121164.549019605,
|
|
"logps/chosen": -497.6589343598055,
|
|
"logps/rejected": -382.6074189291101,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.299715518951416,
|
|
"metrics/advantage_var": 608.7891845703125,
|
|
"regularization/lipschitz_norm": 1587.755859375,
|
|
"regularization/w1": 1.5369476079940796,
|
|
"rewards/chosen": 0.3906628625705911,
|
|
"rewards/margins": 0.7394829922412995,
|
|
"rewards/rejected": -0.3488201296707084,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 19.658206939697266,
|
|
"kl": 13.593878746032715,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -37079324.1722488,
|
|
"logits/rejected": -36729710.16232772,
|
|
"logps/chosen": -501.53902511961724,
|
|
"logps/rejected": -386.3302067381317,
|
|
"loss": 0.6126,
|
|
"loss/base_kto": 3.282827138900757,
|
|
"metrics/advantage_var": 630.2515869140625,
|
|
"regularization/lipschitz_norm": 1671.3150634765625,
|
|
"regularization/w1": 1.6178330183029175,
|
|
"rewards/chosen": 0.414933930173445,
|
|
"rewards/margins": 0.7615973073598683,
|
|
"rewards/rejected": -0.34666337718642326,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 21.1414852142334,
|
|
"kl": 11.913226127624512,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34466315.906976745,
|
|
"logits/rejected": -35772503.080314964,
|
|
"logps/chosen": -476.4670542635659,
|
|
"logps/rejected": -395.8843503937008,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.2135002613067627,
|
|
"metrics/advantage_var": 655.7216186523438,
|
|
"regularization/lipschitz_norm": 1616.7281494140625,
|
|
"regularization/w1": 1.5649927854537964,
|
|
"rewards/chosen": 0.4679276340691618,
|
|
"rewards/margins": 0.8680279576035491,
|
|
"rewards/rejected": -0.4001003235343873,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 19.690521240234375,
|
|
"kl": 16.30158233642578,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35463667.981220655,
|
|
"logits/rejected": -35519796.31825273,
|
|
"logps/chosen": -477.3120109546166,
|
|
"logps/rejected": -366.5620124804992,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.2944581508636475,
|
|
"metrics/advantage_var": 570.8365478515625,
|
|
"regularization/lipschitz_norm": 1522.9305419921875,
|
|
"regularization/w1": 1.4741967916488647,
|
|
"rewards/chosen": 0.5020603902276506,
|
|
"rewards/margins": 0.768465683280193,
|
|
"rewards/rejected": -0.2664052930525424,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.079362869262695,
|
|
"kl": 13.455920219421387,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -36514960.0,
|
|
"logits/rejected": -35912675.2,
|
|
"logps/chosen": -488.150244140625,
|
|
"logps/rejected": -367.4193359375,
|
|
"loss": 0.6099,
|
|
"loss/base_kto": 3.2510571479797363,
|
|
"metrics/advantage_var": 642.3671264648438,
|
|
"regularization/lipschitz_norm": 1682.0211181640625,
|
|
"regularization/w1": 1.628196358680725,
|
|
"rewards/chosen": 0.47579050064086914,
|
|
"rewards/margins": 0.8330754280090332,
|
|
"rewards/rejected": -0.3572849273681641,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 20.961671829223633,
|
|
"kl": 17.843236923217773,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36325404.62111801,
|
|
"logits/rejected": -35787647.19496855,
|
|
"logps/chosen": -474.29745729813664,
|
|
"logps/rejected": -381.296875,
|
|
"loss": 0.6115,
|
|
"loss/base_kto": 3.352402448654175,
|
|
"metrics/advantage_var": 564.89111328125,
|
|
"regularization/lipschitz_norm": 1590.7645263671875,
|
|
"regularization/w1": 1.5398598909378052,
|
|
"rewards/chosen": 0.5228419807386695,
|
|
"rewards/margins": 0.6779734947061236,
|
|
"rewards/rejected": -0.15513151396745406,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 13.721338272094727,
|
|
"kl": 14.79840087890625,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34272889.304615386,
|
|
"logits/rejected": -35379837.155555554,
|
|
"logps/chosen": -495.5737980769231,
|
|
"logps/rejected": -365.0750992063492,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.270128011703491,
|
|
"metrics/advantage_var": 521.1950073242188,
|
|
"regularization/lipschitz_norm": 1532.7447509765625,
|
|
"regularization/w1": 1.4836969375610352,
|
|
"rewards/chosen": 0.51926025390625,
|
|
"rewards/margins": 0.7731879146515377,
|
|
"rewards/rejected": -0.2539276607452877,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 14.794451713562012,
|
|
"kl": 19.552061080932617,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -36663672.23003195,
|
|
"logits/rejected": -37482258.00611621,
|
|
"logps/chosen": -479.8626198083067,
|
|
"logps/rejected": -366.83686926605503,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.298051595687866,
|
|
"metrics/advantage_var": 515.5966186523438,
|
|
"regularization/lipschitz_norm": 1495.6011962890625,
|
|
"regularization/w1": 1.4477418661117554,
|
|
"rewards/chosen": 0.5072054634459864,
|
|
"rewards/margins": 0.7382550273525947,
|
|
"rewards/rejected": -0.23104956390660839,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 17.853073120117188,
|
|
"kl": 14.7052001953125,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -35197236.95178849,
|
|
"logits/rejected": -35854953.29356358,
|
|
"logps/chosen": -496.1546461897356,
|
|
"logps/rejected": -372.1329719387755,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.3330466747283936,
|
|
"metrics/advantage_var": 725.3512573242188,
|
|
"regularization/lipschitz_norm": 1526.4005126953125,
|
|
"regularization/w1": 1.4775556325912476,
|
|
"rewards/chosen": 0.49592269968652797,
|
|
"rewards/margins": 0.7395953086414682,
|
|
"rewards/rejected": -0.24367260895494014,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 14.454861640930176,
|
|
"kl": 15.106024742126465,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34765608.503840245,
|
|
"logits/rejected": -36324043.33014354,
|
|
"logps/chosen": -493.7881144393241,
|
|
"logps/rejected": -363.47831937799043,
|
|
"loss": 0.5661,
|
|
"loss/base_kto": 3.228271961212158,
|
|
"metrics/advantage_var": 470.0264587402344,
|
|
"regularization/lipschitz_norm": 1343.5430908203125,
|
|
"regularization/w1": 1.3005497455596924,
|
|
"rewards/chosen": 0.5775296493975615,
|
|
"rewards/margins": 0.8211360464270671,
|
|
"rewards/rejected": -0.2436063970295056,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 26.791683197021484,
|
|
"kl": 14.966911315917969,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34568147.821263485,
|
|
"logits/rejected": -34942557.312202856,
|
|
"logps/chosen": -476.4884437596302,
|
|
"logps/rejected": -404.7546553090333,
|
|
"loss": 0.5614,
|
|
"loss/base_kto": 3.1534383296966553,
|
|
"metrics/advantage_var": 481.4045104980469,
|
|
"regularization/lipschitz_norm": 1382.3079833984375,
|
|
"regularization/w1": 1.3380740880966187,
|
|
"rewards/chosen": 0.5736989637001637,
|
|
"rewards/margins": 0.9011998526273998,
|
|
"rewards/rejected": -0.32750088892723606,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.575445175170898,
|
|
"kl": 16.950336456298828,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33536975.387442574,
|
|
"logits/rejected": -35163970.55183413,
|
|
"logps/chosen": -492.09652565084224,
|
|
"logps/rejected": -388.9467204944179,
|
|
"loss": 0.5572,
|
|
"loss/base_kto": 3.1143338680267334,
|
|
"metrics/advantage_var": 485.3099670410156,
|
|
"regularization/lipschitz_norm": 1387.7322998046875,
|
|
"regularization/w1": 1.343324899673462,
|
|
"rewards/chosen": 0.5861959880928168,
|
|
"rewards/margins": 0.9141496613384308,
|
|
"rewards/rejected": -0.32795367324561403,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 22.820148468017578,
|
|
"kl": 17.239652633666992,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -36203360.48241206,
|
|
"logits/rejected": -36784128.0,
|
|
"logps/chosen": -480.5009945561139,
|
|
"logps/rejected": -403.6558839677892,
|
|
"loss": 0.5667,
|
|
"loss/base_kto": 3.1273958683013916,
|
|
"metrics/advantage_var": 517.88671875,
|
|
"regularization/lipschitz_norm": 1452.764892578125,
|
|
"regularization/w1": 1.4062763452529907,
|
|
"rewards/chosen": 0.6160600365106784,
|
|
"rewards/margins": 0.9110448232969156,
|
|
"rewards/rejected": -0.2949847867862372,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 14.411775588989258,
|
|
"kl": 8.10576057434082,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33531196.567823343,
|
|
"logits/rejected": -35525020.136222914,
|
|
"logps/chosen": -480.2638505520505,
|
|
"logps/rejected": -365.7645123839009,
|
|
"loss": 0.5609,
|
|
"loss/base_kto": 3.1551380157470703,
|
|
"metrics/advantage_var": 449.5874938964844,
|
|
"regularization/lipschitz_norm": 1376.3428955078125,
|
|
"regularization/w1": 1.332299828529358,
|
|
"rewards/chosen": 0.42819334204640674,
|
|
"rewards/margins": 0.9000151772129363,
|
|
"rewards/rejected": -0.4718218351665296,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 20.173723220825195,
|
|
"kl": 6.088006019592285,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35247337.826771654,
|
|
"logits/rejected": -34913624.50852713,
|
|
"logps/chosen": -481.0663385826772,
|
|
"logps/rejected": -380.60411821705424,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.2020394802093506,
|
|
"metrics/advantage_var": 446.9572448730469,
|
|
"regularization/lipschitz_norm": 1309.5723876953125,
|
|
"regularization/w1": 1.2676661014556885,
|
|
"rewards/chosen": 0.372748763917938,
|
|
"rewards/margins": 0.8560860716335388,
|
|
"rewards/rejected": -0.4833373077156008,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 16.039575576782227,
|
|
"kl": 12.510721206665039,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35488404.14992504,
|
|
"logits/rejected": -37124420.07177814,
|
|
"logps/chosen": -483.5506465517241,
|
|
"logps/rejected": -373.6325958401305,
|
|
"loss": 0.5582,
|
|
"loss/base_kto": 3.146409749984741,
|
|
"metrics/advantage_var": 464.0575256347656,
|
|
"regularization/lipschitz_norm": 1362.4940185546875,
|
|
"regularization/w1": 1.3188942670822144,
|
|
"rewards/chosen": 0.5408348072057722,
|
|
"rewards/margins": 0.929136563226049,
|
|
"rewards/rejected": -0.38830175602027683,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 21.980440139770508,
|
|
"kl": 20.9907283782959,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35655056.83676703,
|
|
"logits/rejected": -35877337.34360555,
|
|
"logps/chosen": -477.984647385103,
|
|
"logps/rejected": -355.39654757318954,
|
|
"loss": 0.5513,
|
|
"loss/base_kto": 3.1715807914733887,
|
|
"metrics/advantage_var": 407.5859375,
|
|
"regularization/lipschitz_norm": 1280.0677490234375,
|
|
"regularization/w1": 1.2391055822372437,
|
|
"rewards/chosen": 0.591469210036772,
|
|
"rewards/margins": 0.8295725740613531,
|
|
"rewards/rejected": -0.2381033640245811,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 19.203744888305664,
|
|
"kl": 14.795648574829102,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33973002.04709576,
|
|
"logits/rejected": -34825838.68118196,
|
|
"logps/chosen": -453.78453689167975,
|
|
"logps/rejected": -356.6351817651633,
|
|
"loss": 0.5511,
|
|
"loss/base_kto": 3.1284186840057373,
|
|
"metrics/advantage_var": 447.4406433105469,
|
|
"regularization/lipschitz_norm": 1323.0413818359375,
|
|
"regularization/w1": 1.280704140663147,
|
|
"rewards/chosen": 0.5981709766238226,
|
|
"rewards/margins": 0.9114278884993086,
|
|
"rewards/rejected": -0.313256911875486,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 26.7326717376709,
|
|
"kl": 11.918484687805176,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33547102.993710693,
|
|
"logits/rejected": -36754610.08695652,
|
|
"logps/chosen": -472.25963050314465,
|
|
"logps/rejected": -375.1269409937888,
|
|
"loss": 0.5592,
|
|
"loss/base_kto": 3.176251173019409,
|
|
"metrics/advantage_var": 467.14892578125,
|
|
"regularization/lipschitz_norm": 1340.0350341796875,
|
|
"regularization/w1": 1.2971538305282593,
|
|
"rewards/chosen": 0.4903901298091097,
|
|
"rewards/margins": 0.8749587647900636,
|
|
"rewards/rejected": -0.384568634980954,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 20.445011138916016,
|
|
"kl": 10.113850593566895,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34621215.33863275,
|
|
"logits/rejected": -36141247.90168971,
|
|
"logps/chosen": -493.0683624801272,
|
|
"logps/rejected": -373.2209101382488,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.1943814754486084,
|
|
"metrics/advantage_var": 494.6817321777344,
|
|
"regularization/lipschitz_norm": 1347.086181640625,
|
|
"regularization/w1": 1.3039792776107788,
|
|
"rewards/chosen": 0.4739505058252186,
|
|
"rewards/margins": 0.8576636494039076,
|
|
"rewards/rejected": -0.383713143578689,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 15.994453430175781,
|
|
"kl": 13.364883422851562,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -36037540.0619195,
|
|
"logits/rejected": -34850266.85173502,
|
|
"logps/chosen": -470.26934984520125,
|
|
"logps/rejected": -391.99669755520506,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.1667163372039795,
|
|
"metrics/advantage_var": 454.56298828125,
|
|
"regularization/lipschitz_norm": 1303.46142578125,
|
|
"regularization/w1": 1.2617506980895996,
|
|
"rewards/chosen": 0.4862136132207817,
|
|
"rewards/margins": 0.8716502639271722,
|
|
"rewards/rejected": -0.3854366507063905,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 14.035725593566895,
|
|
"kl": 15.420308113098145,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34268144.3902439,
|
|
"logits/rejected": -36680356.102564104,
|
|
"logps/chosen": -456.765243902439,
|
|
"logps/rejected": -373.2452423878205,
|
|
"loss": 0.5582,
|
|
"loss/base_kto": 3.2131192684173584,
|
|
"metrics/advantage_var": 402.0440979003906,
|
|
"regularization/lipschitz_norm": 1294.0906982421875,
|
|
"regularization/w1": 1.252679705619812,
|
|
"rewards/chosen": 0.5403972718773818,
|
|
"rewards/margins": 0.8075282458889252,
|
|
"rewards/rejected": -0.26713097401154345,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 15.981244087219238,
|
|
"kl": 23.85441017150879,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36195771.733333334,
|
|
"logits/rejected": -38050386.58064516,
|
|
"logps/chosen": -500.5255681818182,
|
|
"logps/rejected": -361.421622983871,
|
|
"loss": 0.5598,
|
|
"loss/base_kto": 3.1551899909973145,
|
|
"metrics/advantage_var": 439.50146484375,
|
|
"regularization/lipschitz_norm": 1366.6829833984375,
|
|
"regularization/w1": 1.3229492902755737,
|
|
"rewards/chosen": 0.7304390647194602,
|
|
"rewards/margins": 0.8645435657669023,
|
|
"rewards/rejected": -0.13410450104744204,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 16.2845401763916,
|
|
"kl": 22.698949813842773,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35000313.73700306,
|
|
"logits/rejected": -36947228.62619808,
|
|
"logps/chosen": -498.23963111620793,
|
|
"logps/rejected": -393.187250399361,
|
|
"loss": 0.5655,
|
|
"loss/base_kto": 3.1403748989105225,
|
|
"metrics/advantage_var": 477.3729553222656,
|
|
"regularization/lipschitz_norm": 1429.182373046875,
|
|
"regularization/w1": 1.3834484815597534,
|
|
"rewards/chosen": 0.7492074762280199,
|
|
"rewards/margins": 0.8995895699561798,
|
|
"rewards/rejected": -0.15038209372815994,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 14.782350540161133,
|
|
"kl": 17.04379653930664,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35801031.80487805,
|
|
"logits/rejected": -37027925.333333336,
|
|
"logps/chosen": -488.0394912347561,
|
|
"logps/rejected": -348.00971554487177,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.164318084716797,
|
|
"metrics/advantage_var": 466.4730529785156,
|
|
"regularization/lipschitz_norm": 1358.1695556640625,
|
|
"regularization/w1": 1.3147081136703491,
|
|
"rewards/chosen": 0.6286414541849276,
|
|
"rewards/margins": 0.8578833877630275,
|
|
"rewards/rejected": -0.22924193357809997,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 22.164512634277344,
|
|
"kl": 21.0350341796875,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34648109.08176101,
|
|
"logits/rejected": -36091624.149068326,
|
|
"logps/chosen": -470.3832547169811,
|
|
"logps/rejected": -364.16580939440996,
|
|
"loss": 0.5469,
|
|
"loss/base_kto": 3.114414930343628,
|
|
"metrics/advantage_var": 452.1717834472656,
|
|
"regularization/lipschitz_norm": 1302.348876953125,
|
|
"regularization/w1": 1.2606737613677979,
|
|
"rewards/chosen": 0.6658999365080828,
|
|
"rewards/margins": 0.8991428007022064,
|
|
"rewards/rejected": -0.23324286419412363,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 20.64852523803711,
|
|
"kl": 17.060256958007812,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35182916.03072197,
|
|
"logits/rejected": -35944431.72019078,
|
|
"logps/chosen": -479.93514784946234,
|
|
"logps/rejected": -384.7178557233704,
|
|
"loss": 0.5651,
|
|
"loss/base_kto": 3.2080066204071045,
|
|
"metrics/advantage_var": 456.31671142578125,
|
|
"regularization/lipschitz_norm": 1356.2611083984375,
|
|
"regularization/w1": 1.3128607273101807,
|
|
"rewards/chosen": 0.5790699667278706,
|
|
"rewards/margins": 0.8253597036226813,
|
|
"rewards/rejected": -0.2462897368948107,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 13.926713943481445,
|
|
"kl": 15.821423530578613,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33505944.216216218,
|
|
"logits/rejected": -36798584.078175895,
|
|
"logps/chosen": -498.8745307807808,
|
|
"logps/rejected": -378.50712540716614,
|
|
"loss": 0.5532,
|
|
"loss/base_kto": 3.2206246852874756,
|
|
"metrics/advantage_var": 424.4670104980469,
|
|
"regularization/lipschitz_norm": 1244.6424560546875,
|
|
"regularization/w1": 1.204813838005066,
|
|
"rewards/chosen": 0.5900390441711243,
|
|
"rewards/margins": 0.8254382965998066,
|
|
"rewards/rejected": -0.23539925242868232,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 16.446184158325195,
|
|
"kl": 18.047330856323242,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34418546.31446541,
|
|
"logits/rejected": -36722102.85714286,
|
|
"logps/chosen": -460.6459807389937,
|
|
"logps/rejected": -372.9476416925466,
|
|
"loss": 0.5501,
|
|
"loss/base_kto": 3.135580062866211,
|
|
"metrics/advantage_var": 456.362060546875,
|
|
"regularization/lipschitz_norm": 1306.8114013671875,
|
|
"regularization/w1": 1.2649933099746704,
|
|
"rewards/chosen": 0.6586240852403941,
|
|
"rewards/margins": 0.9100232024673205,
|
|
"rewards/rejected": -0.2513991172269264,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 17.3842716217041,
|
|
"kl": 17.253808975219727,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33967684.48407643,
|
|
"logits/rejected": -34660094.42944785,
|
|
"logps/chosen": -459.5206011146497,
|
|
"logps/rejected": -356.4508243865031,
|
|
"loss": 0.5537,
|
|
"loss/base_kto": 3.123319149017334,
|
|
"metrics/advantage_var": 463.88824462890625,
|
|
"regularization/lipschitz_norm": 1349.2430419921875,
|
|
"regularization/w1": 1.3060671091079712,
|
|
"rewards/chosen": 0.6501993555931529,
|
|
"rewards/margins": 0.9303998223810411,
|
|
"rewards/rejected": -0.28020046678788824,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 15.38463306427002,
|
|
"kl": 17.004545211791992,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -36372691.53051643,
|
|
"logits/rejected": -37632182.115444615,
|
|
"logps/chosen": -495.6032863849765,
|
|
"logps/rejected": -352.78276131045243,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.1340577602386475,
|
|
"metrics/advantage_var": 473.86279296875,
|
|
"regularization/lipschitz_norm": 1348.970703125,
|
|
"regularization/w1": 1.3058035373687744,
|
|
"rewards/chosen": 0.6435212089049052,
|
|
"rewards/margins": 0.9046952129939749,
|
|
"rewards/rejected": -0.2611740040890698,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 19.00291633605957,
|
|
"kl": 19.351795196533203,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -36163217.2972973,
|
|
"logits/rejected": -36807031.069767445,
|
|
"logps/chosen": -471.9927153716216,
|
|
"logps/rejected": -411.79137899709303,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.1698896884918213,
|
|
"metrics/advantage_var": 477.393310546875,
|
|
"regularization/lipschitz_norm": 1332.4852294921875,
|
|
"regularization/w1": 1.2898457050323486,
|
|
"rewards/chosen": 0.5332078675966005,
|
|
"rewards/margins": 0.8690700938610219,
|
|
"rewards/rejected": -0.3358622262644213,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 14.313151359558105,
|
|
"kl": 15.304774284362793,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35561039.50310559,
|
|
"logits/rejected": -36505094.440251574,
|
|
"logps/chosen": -485.09666149068323,
|
|
"logps/rejected": -375.82920597484275,
|
|
"loss": 0.5602,
|
|
"loss/base_kto": 3.1851680278778076,
|
|
"metrics/advantage_var": 407.154052734375,
|
|
"regularization/lipschitz_norm": 1339.4749755859375,
|
|
"regularization/w1": 1.2966117858886719,
|
|
"rewards/chosen": 0.5668783720976077,
|
|
"rewards/margins": 0.8416969604504109,
|
|
"rewards/rejected": -0.27481858835280315,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 13.861672401428223,
|
|
"kl": 16.457523345947266,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35199669.624398075,
|
|
"logits/rejected": -36202736.02435312,
|
|
"logps/chosen": -487.6015248796148,
|
|
"logps/rejected": -384.9729832572298,
|
|
"loss": 0.5528,
|
|
"loss/base_kto": 3.1752567291259766,
|
|
"metrics/advantage_var": 440.41851806640625,
|
|
"regularization/lipschitz_norm": 1288.4154052734375,
|
|
"regularization/w1": 1.2471860647201538,
|
|
"rewards/chosen": 0.5853135306418038,
|
|
"rewards/margins": 0.860222609625941,
|
|
"rewards/rejected": -0.2749090789841372,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 16.641489028930664,
|
|
"kl": 14.819449424743652,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -36549506.96754251,
|
|
"logits/rejected": -36979923.9178515,
|
|
"logps/chosen": -512.8835007727976,
|
|
"logps/rejected": -367.8369618878357,
|
|
"loss": 0.5511,
|
|
"loss/base_kto": 3.153918504714966,
|
|
"metrics/advantage_var": 458.3197937011719,
|
|
"regularization/lipschitz_norm": 1296.5853271484375,
|
|
"regularization/w1": 1.2550947666168213,
|
|
"rewards/chosen": 0.560027557325879,
|
|
"rewards/margins": 0.9050760007269083,
|
|
"rewards/rejected": -0.3450484434010293,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 19.75300407409668,
|
|
"kl": 15.24942684173584,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -37194720.0,
|
|
"logits/rejected": -36386256.0,
|
|
"logps/chosen": -486.0267578125,
|
|
"logps/rejected": -376.722021484375,
|
|
"loss": 0.5595,
|
|
"loss/base_kto": 3.1709554195404053,
|
|
"metrics/advantage_var": 410.4849548339844,
|
|
"regularization/lipschitz_norm": 1348.0032958984375,
|
|
"regularization/w1": 1.304867148399353,
|
|
"rewards/chosen": 0.5386874198913574,
|
|
"rewards/margins": 0.8349444150924682,
|
|
"rewards/rejected": -0.2962569952011108,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 19.20242691040039,
|
|
"kl": 15.10105037689209,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35009629.09090909,
|
|
"logits/rejected": -36506374.66462481,
|
|
"logps/chosen": -489.31419457735245,
|
|
"logps/rejected": -364.37267898162327,
|
|
"loss": 0.5498,
|
|
"loss/base_kto": 3.130419969558716,
|
|
"metrics/advantage_var": 406.90325927734375,
|
|
"regularization/lipschitz_norm": 1309.8929443359375,
|
|
"regularization/w1": 1.2679764032363892,
|
|
"rewards/chosen": 0.5749126720276365,
|
|
"rewards/margins": 0.8808706239457668,
|
|
"rewards/rejected": -0.30595795191813024,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 19.256906509399414,
|
|
"kl": 12.742453575134277,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -33597146.04416404,
|
|
"logits/rejected": -36226650.35294118,
|
|
"logps/chosen": -458.092665615142,
|
|
"logps/rejected": -383.2747678018576,
|
|
"loss": 0.561,
|
|
"loss/base_kto": 3.1785125732421875,
|
|
"metrics/advantage_var": 452.4025573730469,
|
|
"regularization/lipschitz_norm": 1353.0130615234375,
|
|
"regularization/w1": 1.3097165822982788,
|
|
"rewards/chosen": 0.5305636433020752,
|
|
"rewards/margins": 0.8693291685294359,
|
|
"rewards/rejected": -0.3387655252273607,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.791271209716797,
|
|
"kl": 14.284051895141602,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -36129347.2,
|
|
"logits/rejected": -36142502.4,
|
|
"logps/chosen": -468.863916015625,
|
|
"logps/rejected": -400.61025390625,
|
|
"loss": 0.5533,
|
|
"loss/base_kto": 3.1291942596435547,
|
|
"metrics/advantage_var": 468.5518493652344,
|
|
"regularization/lipschitz_norm": 1339.9281005859375,
|
|
"regularization/w1": 1.2970503568649292,
|
|
"rewards/chosen": 0.5568619728088379,
|
|
"rewards/margins": 0.9175825119018555,
|
|
"rewards/rejected": -0.3607205390930176,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 18.03066635131836,
|
|
"kl": 13.904191970825195,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34961040.41025641,
|
|
"logits/rejected": -36199255.414634146,
|
|
"logps/chosen": -491.5910456730769,
|
|
"logps/rejected": -377.5545445884146,
|
|
"loss": 0.5549,
|
|
"loss/base_kto": 3.1545772552490234,
|
|
"metrics/advantage_var": 450.41845703125,
|
|
"regularization/lipschitz_norm": 1327.04248046875,
|
|
"regularization/w1": 1.2845771312713623,
|
|
"rewards/chosen": 0.5091549799992487,
|
|
"rewards/margins": 0.9020983855227815,
|
|
"rewards/rejected": -0.3929434055235328,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 14.523098945617676,
|
|
"kl": 15.337979316711426,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34310478.89308176,
|
|
"logits/rejected": -36039253.863354035,
|
|
"logps/chosen": -471.1668632075472,
|
|
"logps/rejected": -371.7253251164596,
|
|
"loss": 0.5691,
|
|
"loss/base_kto": 3.219719648361206,
|
|
"metrics/advantage_var": 450.0986328125,
|
|
"regularization/lipschitz_norm": 1377.5322265625,
|
|
"regularization/w1": 1.3334511518478394,
|
|
"rewards/chosen": 0.4841920384820902,
|
|
"rewards/margins": 0.8305825434968297,
|
|
"rewards/rejected": -0.34639050501473945,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 16.3944091796875,
|
|
"kl": 13.576468467712402,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34964277.788309634,
|
|
"logits/rejected": -35085031.86398763,
|
|
"logps/chosen": -480.72600710900474,
|
|
"logps/rejected": -380.27637171561054,
|
|
"loss": 0.5548,
|
|
"loss/base_kto": 3.151641845703125,
|
|
"metrics/advantage_var": 433.5982360839844,
|
|
"regularization/lipschitz_norm": 1328.992919921875,
|
|
"regularization/w1": 1.286465048789978,
|
|
"rewards/chosen": 0.5746732835332741,
|
|
"rewards/margins": 0.8706274026422023,
|
|
"rewards/rejected": -0.2959541191089282,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 15.026321411132812,
|
|
"kl": 10.481276512145996,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -35691532.449848026,
|
|
"logits/rejected": -34861253.5562701,
|
|
"logps/chosen": -478.09902165653494,
|
|
"logps/rejected": -376.7376657958199,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.2123265266418457,
|
|
"metrics/advantage_var": 453.0511779785156,
|
|
"regularization/lipschitz_norm": 1331.392822265625,
|
|
"regularization/w1": 1.2887881994247437,
|
|
"rewards/chosen": 0.530893435956497,
|
|
"rewards/margins": 0.8606501941772063,
|
|
"rewards/rejected": -0.3297567582207094,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 17.59103775024414,
|
|
"kl": 15.0452880859375,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36011022.467817895,
|
|
"logits/rejected": -37394725.02643857,
|
|
"logps/chosen": -493.9991169544741,
|
|
"logps/rejected": -364.99878499222393,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.215874433517456,
|
|
"metrics/advantage_var": 424.0643615722656,
|
|
"regularization/lipschitz_norm": 1326.4677734375,
|
|
"regularization/w1": 1.2840207815170288,
|
|
"rewards/chosen": 0.5065736875429259,
|
|
"rewards/margins": 0.8204087247981018,
|
|
"rewards/rejected": -0.3138350372551759,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 18.746498107910156,
|
|
"kl": 13.1990966796875,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -36079173.18918919,
|
|
"logits/rejected": -36341291.256528415,
|
|
"logps/chosen": -480.635532591415,
|
|
"logps/rejected": -389.09888632872503,
|
|
"loss": 0.5585,
|
|
"loss/base_kto": 3.1526682376861572,
|
|
"metrics/advantage_var": 444.0506896972656,
|
|
"regularization/lipschitz_norm": 1358.612548828125,
|
|
"regularization/w1": 1.3151369094848633,
|
|
"rewards/chosen": 0.5417781763122268,
|
|
"rewards/margins": 0.896477681768076,
|
|
"rewards/rejected": -0.35469950545584916,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 13.928918838500977,
|
|
"kl": 16.49699592590332,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -35727701.333333336,
|
|
"logits/rejected": -36388502.22754491,
|
|
"logps/chosen": -461.0346200980392,
|
|
"logps/rejected": -371.1039015718563,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.1694607734680176,
|
|
"metrics/advantage_var": 453.6496276855469,
|
|
"regularization/lipschitz_norm": 1312.77734375,
|
|
"regularization/w1": 1.270768404006958,
|
|
"rewards/chosen": 0.5379319533803104,
|
|
"rewards/margins": 0.8702755973911169,
|
|
"rewards/rejected": -0.3323436440108065,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 17.58167839050293,
|
|
"kl": 10.382193565368652,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -35481813.333333336,
|
|
"logits/rejected": -36820789.89473684,
|
|
"logps/chosen": -502.3920665922619,
|
|
"logps/rejected": -358.97743626644734,
|
|
"loss": 0.5541,
|
|
"loss/base_kto": 3.197376012802124,
|
|
"metrics/advantage_var": 425.8365783691406,
|
|
"regularization/lipschitz_norm": 1275.8798828125,
|
|
"regularization/w1": 1.2350517511367798,
|
|
"rewards/chosen": 0.5288776670183454,
|
|
"rewards/margins": 0.8674364412637581,
|
|
"rewards/rejected": -0.33855877424541275,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_loss": 0.5859840150941839,
|
|
"train_runtime": 11024.7736,
|
|
"train_samples_per_second": 13.444,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|