2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 22.15345001220703,
|
|
"kl": 13.118853569030762,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36892383.057364345,
|
|
"logits/rejected": -37261940.91338582,
|
|
"logps/chosen": -498.7265503875969,
|
|
"logps/rejected": -360.81296751968506,
|
|
"loss": 0.6132,
|
|
"loss/base_kto": 3.937687635421753,
|
|
"metrics/advantage_var": 216.78453063964844,
|
|
"regularization/lipschitz_norm": 1000.3004150390625,
|
|
"regularization/w1": 0.9682908058166504,
|
|
"rewards/chosen": 0.18708436951156734,
|
|
"rewards/margins": 0.06643253566944751,
|
|
"rewards/rejected": 0.12065183384211983,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 23.479450225830078,
|
|
"kl": 10.830110549926758,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35865847.3883792,
|
|
"logits/rejected": -37753666.249201275,
|
|
"logps/chosen": -485.23040902140673,
|
|
"logps/rejected": -375.0230630990415,
|
|
"loss": 0.6081,
|
|
"loss/base_kto": 3.9517877101898193,
|
|
"metrics/advantage_var": 197.56771850585938,
|
|
"regularization/lipschitz_norm": 942.9439697265625,
|
|
"regularization/w1": 0.9127697348594666,
|
|
"rewards/chosen": 0.1295515171249343,
|
|
"rewards/margins": 0.02920465023695505,
|
|
"rewards/rejected": 0.10034686688797924,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 28.262170791625977,
|
|
"kl": 21.241575241088867,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36815843.81651376,
|
|
"logits/rejected": -37640237.801916935,
|
|
"logps/chosen": -466.9399369266055,
|
|
"logps/rejected": -351.68540335463257,
|
|
"loss": 0.5865,
|
|
"loss/base_kto": 3.848869800567627,
|
|
"metrics/advantage_var": 172.2936553955078,
|
|
"regularization/lipschitz_norm": 870.8187866210938,
|
|
"regularization/w1": 0.84295254945755,
|
|
"rewards/chosen": 0.3665013677847859,
|
|
"rewards/margins": 0.14264552346382448,
|
|
"rewards/rejected": 0.22385584432096145,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 37.89582443237305,
|
|
"kl": 10.080211639404297,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36627655.84266263,
|
|
"logits/rejected": -37849772.872374795,
|
|
"logps/chosen": -501.98988275340395,
|
|
"logps/rejected": -382.7355361470113,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.831226110458374,
|
|
"metrics/advantage_var": 209.652099609375,
|
|
"regularization/lipschitz_norm": 977.0872192382812,
|
|
"regularization/w1": 0.9458203315734863,
|
|
"rewards/chosen": 0.19208223051454945,
|
|
"rewards/margins": 0.15911251075564523,
|
|
"rewards/rejected": 0.03296971975890423,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 29.207448959350586,
|
|
"kl": 15.843803405761719,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35941613.307936504,
|
|
"logits/rejected": -38354969.20615385,
|
|
"logps/chosen": -494.08730158730157,
|
|
"logps/rejected": -359.3373076923077,
|
|
"loss": 0.5993,
|
|
"loss/base_kto": 3.8869011402130127,
|
|
"metrics/advantage_var": 217.6295623779297,
|
|
"regularization/lipschitz_norm": 937.18994140625,
|
|
"regularization/w1": 0.9071998596191406,
|
|
"rewards/chosen": 0.2710543677920387,
|
|
"rewards/margins": 0.10979380401499542,
|
|
"rewards/rejected": 0.16126056377704326,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 16.20916748046875,
|
|
"kl": 28.23166847229004,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36104425.08042488,
|
|
"logits/rejected": -38280140.88244767,
|
|
"logps/chosen": -479.24506828528075,
|
|
"logps/rejected": -350.1025311996779,
|
|
"loss": 0.5918,
|
|
"loss/base_kto": 3.763122320175171,
|
|
"metrics/advantage_var": 232.70542907714844,
|
|
"regularization/lipschitz_norm": 1003.3552856445312,
|
|
"regularization/w1": 0.9712478518486023,
|
|
"rewards/chosen": 0.49239121293804533,
|
|
"rewards/margins": 0.18967615099473312,
|
|
"rewards/rejected": 0.3027150619433122,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 49.2159423828125,
|
|
"kl": 19.661544799804688,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36235399.67192429,
|
|
"logits/rejected": -38335624.32198142,
|
|
"logps/chosen": -488.79303036277605,
|
|
"logps/rejected": -386.08644543343655,
|
|
"loss": 0.5968,
|
|
"loss/base_kto": 3.8157060146331787,
|
|
"metrics/advantage_var": 230.65576171875,
|
|
"regularization/lipschitz_norm": 990.57861328125,
|
|
"regularization/w1": 0.9588800668716431,
|
|
"rewards/chosen": 0.27197872125764,
|
|
"rewards/margins": 0.15675797523796364,
|
|
"rewards/rejected": 0.11522074601967637,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 19.986955642700195,
|
|
"kl": 3.6723923683166504,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35298153.8583196,
|
|
"logits/rejected": -37381644.93313522,
|
|
"logps/chosen": -493.0083916803954,
|
|
"logps/rejected": -384.68931092124814,
|
|
"loss": 0.6018,
|
|
"loss/base_kto": 3.864077091217041,
|
|
"metrics/advantage_var": 210.59732055664062,
|
|
"regularization/lipschitz_norm": 981.7514038085938,
|
|
"regularization/w1": 0.9503352046012878,
|
|
"rewards/chosen": -0.08236450496968956,
|
|
"rewards/margins": 0.11254791094145922,
|
|
"rewards/rejected": -0.19491241591114877,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.850082397460938,
|
|
"kl": 4.33528470993042,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -37214159.23809524,
|
|
"logits/rejected": -38470240.09846154,
|
|
"logps/chosen": -507.22728174603174,
|
|
"logps/rejected": -377.2554326923077,
|
|
"loss": 0.602,
|
|
"loss/base_kto": 3.815427541732788,
|
|
"metrics/advantage_var": 230.77696228027344,
|
|
"regularization/lipschitz_norm": 1033.9093017578125,
|
|
"regularization/w1": 1.000824213027954,
|
|
"rewards/chosen": 0.003390933596898639,
|
|
"rewards/margins": 0.1816865620712075,
|
|
"rewards/rejected": -0.17829562847430888,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 23.678598403930664,
|
|
"kl": 3.3330790996551514,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36878955.78947368,
|
|
"logits/rejected": -38087161.9047619,
|
|
"logps/chosen": -487.76089638157896,
|
|
"logps/rejected": -393.0424107142857,
|
|
"loss": 0.5922,
|
|
"loss/base_kto": 3.786081075668335,
|
|
"metrics/advantage_var": 234.61083984375,
|
|
"regularization/lipschitz_norm": 983.1530151367188,
|
|
"regularization/w1": 0.9516921043395996,
|
|
"rewards/chosen": -0.07404754663768567,
|
|
"rewards/margins": 0.18513093287484686,
|
|
"rewards/rejected": -0.25917847951253253,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 16.462984085083008,
|
|
"kl": 2.6557037830352783,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -34828644.4556962,
|
|
"logits/rejected": -36343902.81481481,
|
|
"logps/chosen": -497.4192049050633,
|
|
"logps/rejected": -389.8587962962963,
|
|
"loss": 0.5948,
|
|
"loss/base_kto": 3.772526502609253,
|
|
"metrics/advantage_var": 230.02403259277344,
|
|
"regularization/lipschitz_norm": 1018.7216796875,
|
|
"regularization/w1": 0.9861226081848145,
|
|
"rewards/chosen": -0.06652461426167548,
|
|
"rewards/margins": 0.22798612114414046,
|
|
"rewards/rejected": -0.29451073540581596,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 22.18965721130371,
|
|
"kl": 11.050432205200195,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -36185746.71345029,
|
|
"logits/rejected": -37877464.48322148,
|
|
"logps/chosen": -504.24858369883043,
|
|
"logps/rejected": -354.30141044463085,
|
|
"loss": 0.5948,
|
|
"loss/base_kto": 3.7301831245422363,
|
|
"metrics/advantage_var": 254.15298461914062,
|
|
"regularization/lipschitz_norm": 1061.8812255859375,
|
|
"regularization/w1": 1.02790105342865,
|
|
"rewards/chosen": 0.23639009709943803,
|
|
"rewards/margins": 0.25230320755084834,
|
|
"rewards/rejected": -0.015913110451410282,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.791778564453125,
|
|
"kl": 16.512481689453125,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -35357778.17283951,
|
|
"logits/rejected": -36711543.898734175,
|
|
"logps/chosen": -484.9001736111111,
|
|
"logps/rejected": -349.962890625,
|
|
"loss": 0.5992,
|
|
"loss/base_kto": 3.768084764480591,
|
|
"metrics/advantage_var": 263.63470458984375,
|
|
"regularization/lipschitz_norm": 1059.4764404296875,
|
|
"regularization/w1": 1.0255731344223022,
|
|
"rewards/chosen": 0.2958780453528887,
|
|
"rewards/margins": 0.20420164718425243,
|
|
"rewards/rejected": 0.09167639816863628,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 22.197799682617188,
|
|
"kl": 5.907263278961182,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36133134.61663948,
|
|
"logits/rejected": -38496965.27736132,
|
|
"logps/chosen": -490.0870717781403,
|
|
"logps/rejected": -366.56552192653675,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.7622933387756348,
|
|
"metrics/advantage_var": 244.79617309570312,
|
|
"regularization/lipschitz_norm": 1043.1629638671875,
|
|
"regularization/w1": 1.0097817182540894,
|
|
"rewards/chosen": 0.015458139665364053,
|
|
"rewards/margins": 0.20506181652566438,
|
|
"rewards/rejected": -0.18960367686030033,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 18.498868942260742,
|
|
"kl": 11.734432220458984,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35139068.8,
|
|
"logits/rejected": -36400768.0,
|
|
"logps/chosen": -495.54189453125,
|
|
"logps/rejected": -379.73623046875,
|
|
"loss": 0.5941,
|
|
"loss/base_kto": 3.75775408744812,
|
|
"metrics/advantage_var": 235.85008239746094,
|
|
"regularization/lipschitz_norm": 1027.7601318359375,
|
|
"regularization/w1": 0.9948717951774597,
|
|
"rewards/chosen": 0.20426039695739745,
|
|
"rewards/margins": 0.2161217242479324,
|
|
"rewards/rejected": -0.011861327290534972,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 23.94541358947754,
|
|
"kl": 9.383148193359375,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36406463.12594841,
|
|
"logits/rejected": -36544749.44927536,
|
|
"logps/chosen": -457.58568854324733,
|
|
"logps/rejected": -376.09704609500807,
|
|
"loss": 0.5933,
|
|
"loss/base_kto": 3.7974541187286377,
|
|
"metrics/advantage_var": 217.3662109375,
|
|
"regularization/lipschitz_norm": 980.08740234375,
|
|
"regularization/w1": 0.9487245678901672,
|
|
"rewards/chosen": 0.14072488833993266,
|
|
"rewards/margins": 0.1926741482714632,
|
|
"rewards/rejected": -0.051949259931530545,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 15.679771423339844,
|
|
"kl": 9.631284713745117,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36625063.476635516,
|
|
"logits/rejected": -37321076.36363637,
|
|
"logps/chosen": -474.5097352024922,
|
|
"logps/rejected": -384.68387539184954,
|
|
"loss": 0.595,
|
|
"loss/base_kto": 3.7828240394592285,
|
|
"metrics/advantage_var": 215.48794555664062,
|
|
"regularization/lipschitz_norm": 1009.1638793945312,
|
|
"regularization/w1": 0.9768705368041992,
|
|
"rewards/chosen": 0.1625291105371398,
|
|
"rewards/margins": 0.2038068943286131,
|
|
"rewards/rejected": -0.041277783791473295,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 19.622116088867188,
|
|
"kl": 9.291582107543945,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36896465.41835148,
|
|
"logits/rejected": -37270082.71271586,
|
|
"logps/chosen": -493.0998250388802,
|
|
"logps/rejected": -381.175431711146,
|
|
"loss": 0.5927,
|
|
"loss/base_kto": 3.7375876903533936,
|
|
"metrics/advantage_var": 233.1632080078125,
|
|
"regularization/lipschitz_norm": 1036.80126953125,
|
|
"regularization/w1": 1.003623604774475,
|
|
"rewards/chosen": 0.17569185675210197,
|
|
"rewards/margins": 0.26097282759052,
|
|
"rewards/rejected": -0.085280970838418,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 21.20962142944336,
|
|
"kl": 5.696521282196045,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36369843.2,
|
|
"logits/rejected": -37739568.0,
|
|
"logps/chosen": -501.78984375,
|
|
"logps/rejected": -370.9171142578125,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.7792370319366455,
|
|
"metrics/advantage_var": 230.82839965820312,
|
|
"regularization/lipschitz_norm": 1007.4281616210938,
|
|
"regularization/w1": 0.9751904606819153,
|
|
"rewards/chosen": 0.03979195356369018,
|
|
"rewards/margins": 0.21980226039886475,
|
|
"rewards/rejected": -0.18001030683517455,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 21.195293426513672,
|
|
"kl": 11.104262351989746,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35798316.78972713,
|
|
"logits/rejected": -37209516.61491629,
|
|
"logps/chosen": -495.0862760834671,
|
|
"logps/rejected": -374.8033200152207,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.777482748031616,
|
|
"metrics/advantage_var": 221.63101196289062,
|
|
"regularization/lipschitz_norm": 991.3137817382812,
|
|
"regularization/w1": 0.959591805934906,
|
|
"rewards/chosen": 0.13961446648807435,
|
|
"rewards/margins": 0.2012568273204861,
|
|
"rewards/rejected": -0.061642360832411765,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 16.181947708129883,
|
|
"kl": 7.641064643859863,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35405199.97406807,
|
|
"logits/rejected": -35969447.19155355,
|
|
"logps/chosen": -483.6488047001621,
|
|
"logps/rejected": -374.25,
|
|
"loss": 0.6023,
|
|
"loss/base_kto": 3.783252716064453,
|
|
"metrics/advantage_var": 263.3688659667969,
|
|
"regularization/lipschitz_norm": 1069.1710205078125,
|
|
"regularization/w1": 1.0349575281143188,
|
|
"rewards/chosen": 0.07617089814073262,
|
|
"rewards/margins": 0.1918000799234968,
|
|
"rewards/rejected": -0.11562918178276418,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 18.227420806884766,
|
|
"kl": 1.661803126335144,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35387650.42271294,
|
|
"logits/rejected": -36372454.6377709,
|
|
"logps/chosen": -499.09996056782336,
|
|
"logps/rejected": -365.34302438080493,
|
|
"loss": 0.6021,
|
|
"loss/base_kto": 3.7776076793670654,
|
|
"metrics/advantage_var": 251.21963500976562,
|
|
"regularization/lipschitz_norm": 1073.647216796875,
|
|
"regularization/w1": 1.039290428161621,
|
|
"rewards/chosen": -0.24246634594649547,
|
|
"rewards/margins": 0.22182289115562337,
|
|
"rewards/rejected": -0.46428923710211883,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 24.066246032714844,
|
|
"kl": 6.8650031089782715,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35609759.89368771,
|
|
"logits/rejected": -36933622.938053094,
|
|
"logps/chosen": -477.5497819767442,
|
|
"logps/rejected": -372.4376843657817,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.754575490951538,
|
|
"metrics/advantage_var": 251.0904998779297,
|
|
"regularization/lipschitz_norm": 1047.5648193359375,
|
|
"regularization/w1": 1.014042854309082,
|
|
"rewards/chosen": 0.06181729592358155,
|
|
"rewards/margins": 0.22061771924470064,
|
|
"rewards/rejected": -0.1588004233211191,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 18.91375732421875,
|
|
"kl": 5.622692108154297,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -34431138.58931298,
|
|
"logits/rejected": -34823720.1408,
|
|
"logps/chosen": -457.42996183206105,
|
|
"logps/rejected": -370.341,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.82047438621521,
|
|
"metrics/advantage_var": 306.2411193847656,
|
|
"regularization/lipschitz_norm": 1062.985595703125,
|
|
"regularization/w1": 1.0289700031280518,
|
|
"rewards/chosen": 0.0025175647881194836,
|
|
"rewards/margins": 0.1897200061943695,
|
|
"rewards/rejected": -0.18720244140625,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 26.11575698852539,
|
|
"kl": 16.56481170654297,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35165868.254263565,
|
|
"logits/rejected": -34430076.17007874,
|
|
"logps/chosen": -474.0483042635659,
|
|
"logps/rejected": -372.79271653543304,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.707665205001831,
|
|
"metrics/advantage_var": 275.8280334472656,
|
|
"regularization/lipschitz_norm": 1106.2689208984375,
|
|
"regularization/w1": 1.0708683729171753,
|
|
"rewards/chosen": 0.36971151662427326,
|
|
"rewards/margins": 0.24955291412750286,
|
|
"rewards/rejected": 0.12015860249677042,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 16.84819221496582,
|
|
"kl": 12.027435302734375,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -34673724.04437401,
|
|
"logits/rejected": -35028297.76271187,
|
|
"logps/chosen": -449.09974247226626,
|
|
"logps/rejected": -333.949345146379,
|
|
"loss": 0.5766,
|
|
"loss/base_kto": 3.631495714187622,
|
|
"metrics/advantage_var": 246.57864379882812,
|
|
"regularization/lipschitz_norm": 1013.8653564453125,
|
|
"regularization/w1": 0.9814216494560242,
|
|
"rewards/chosen": 0.2777647329774787,
|
|
"rewards/margins": 0.36490735991179474,
|
|
"rewards/rejected": -0.08714262693431601,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 25.673999786376953,
|
|
"kl": 4.097499370574951,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -34443241.13556619,
|
|
"logits/rejected": -35671668.826952524,
|
|
"logps/chosen": -470.14179625199364,
|
|
"logps/rejected": -388.5231144716692,
|
|
"loss": 0.6012,
|
|
"loss/base_kto": 3.788609266281128,
|
|
"metrics/advantage_var": 261.6991882324219,
|
|
"regularization/lipschitz_norm": 1055.010009765625,
|
|
"regularization/w1": 1.0212496519088745,
|
|
"rewards/chosen": -0.13218634208423669,
|
|
"rewards/margins": 0.21239389493780103,
|
|
"rewards/rejected": -0.3445802370220377,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 20.85354232788086,
|
|
"kl": 5.644055366516113,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -34549834.135746606,
|
|
"logits/rejected": -35782642.72285251,
|
|
"logps/chosen": -495.15813536953243,
|
|
"logps/rejected": -374.7583063209076,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.703131914138794,
|
|
"metrics/advantage_var": 281.5477294921875,
|
|
"regularization/lipschitz_norm": 1094.409423828125,
|
|
"regularization/w1": 1.0593883991241455,
|
|
"rewards/chosen": 0.040104973909542034,
|
|
"rewards/margins": 0.30046022621720514,
|
|
"rewards/rejected": -0.2603552523076631,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 26.437442779541016,
|
|
"kl": 11.23274040222168,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -37483499.921568625,
|
|
"logits/rejected": -37331262.651539706,
|
|
"logps/chosen": -508.5079185520362,
|
|
"logps/rejected": -383.1639232171799,
|
|
"loss": 0.5962,
|
|
"loss/base_kto": 3.7372078895568848,
|
|
"metrics/advantage_var": 253.04470825195312,
|
|
"regularization/lipschitz_norm": 1066.7435302734375,
|
|
"regularization/w1": 1.0326077938079834,
|
|
"rewards/chosen": 0.22099568869013952,
|
|
"rewards/margins": 0.24528079185587578,
|
|
"rewards/rejected": -0.024285103165736267,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 19.478317260742188,
|
|
"kl": 17.681842803955078,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -35754006.5408805,
|
|
"logits/rejected": -37241251.77639752,
|
|
"logps/chosen": -493.9110652515723,
|
|
"logps/rejected": -354.98311335403724,
|
|
"loss": 0.5931,
|
|
"loss/base_kto": 3.7448461055755615,
|
|
"metrics/advantage_var": 254.2628936767578,
|
|
"regularization/lipschitz_norm": 1032.640869140625,
|
|
"regularization/w1": 0.9995962977409363,
|
|
"rewards/chosen": 0.2779609392274101,
|
|
"rewards/margins": 0.23731708135440568,
|
|
"rewards/rejected": 0.04064385787300442,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 18.940183639526367,
|
|
"kl": 15.43096923828125,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -35199862.15384615,
|
|
"logits/rejected": -36608480.780487806,
|
|
"logps/chosen": -465.35281450320514,
|
|
"logps/rejected": -383.1649914253049,
|
|
"loss": 0.5963,
|
|
"loss/base_kto": 3.7277181148529053,
|
|
"metrics/advantage_var": 249.8613739013672,
|
|
"regularization/lipschitz_norm": 1076.76611328125,
|
|
"regularization/w1": 1.0423096418380737,
|
|
"rewards/chosen": 0.1592861811319987,
|
|
"rewards/margins": 0.2374682736590626,
|
|
"rewards/rejected": -0.07818209252706389,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 20.173166275024414,
|
|
"kl": 13.661532402038574,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -36712131.93070866,
|
|
"logits/rejected": -37734060.254263565,
|
|
"logps/chosen": -482.89995078740156,
|
|
"logps/rejected": -385.07577519379845,
|
|
"loss": 0.5843,
|
|
"loss/base_kto": 3.683251142501831,
|
|
"metrics/advantage_var": 242.1837158203125,
|
|
"regularization/lipschitz_norm": 1023.6516723632812,
|
|
"regularization/w1": 0.9908949136734009,
|
|
"rewards/chosen": 0.2188158891332431,
|
|
"rewards/margins": 0.2773357286344392,
|
|
"rewards/rejected": -0.0585198395011961,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 25.639448165893555,
|
|
"kl": 14.475909233093262,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -37584781.361573376,
|
|
"logits/rejected": -37655454.39741518,
|
|
"logps/chosen": -479.821482602118,
|
|
"logps/rejected": -382.38734349757675,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.7747390270233154,
|
|
"metrics/advantage_var": 260.9999084472656,
|
|
"regularization/lipschitz_norm": 1050.651611328125,
|
|
"regularization/w1": 1.0170307159423828,
|
|
"rewards/chosen": 0.2322255190850747,
|
|
"rewards/margins": 0.19899130337795823,
|
|
"rewards/rejected": 0.033234215707116446,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 29.733293533325195,
|
|
"kl": 14.45817756652832,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36410332.45619835,
|
|
"logits/rejected": -37131957.285925925,
|
|
"logps/chosen": -462.92060950413224,
|
|
"logps/rejected": -371.6128009259259,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.725926160812378,
|
|
"metrics/advantage_var": 262.250732421875,
|
|
"regularization/lipschitz_norm": 1027.5491943359375,
|
|
"regularization/w1": 0.994667649269104,
|
|
"rewards/chosen": 0.21567261751033057,
|
|
"rewards/margins": 0.2070352160708051,
|
|
"rewards/rejected": 0.008637401439525462,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 18.668148040771484,
|
|
"kl": 10.823233604431152,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36379290.93701997,
|
|
"logits/rejected": -37690099.38314785,
|
|
"logps/chosen": -487.93970814132103,
|
|
"logps/rejected": -384.5054153418124,
|
|
"loss": 0.5936,
|
|
"loss/base_kto": 3.717161178588867,
|
|
"metrics/advantage_var": 274.3131408691406,
|
|
"regularization/lipschitz_norm": 1065.49609375,
|
|
"regularization/w1": 1.031400203704834,
|
|
"rewards/chosen": 0.10203624906994048,
|
|
"rewards/margins": 0.2453869927867156,
|
|
"rewards/rejected": -0.14335074371677514,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 24.702688217163086,
|
|
"kl": 17.650907516479492,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -37009155.16049383,
|
|
"logits/rejected": -37596143.79746836,
|
|
"logps/chosen": -504.19675925925924,
|
|
"logps/rejected": -377.22868868670884,
|
|
"loss": 0.5831,
|
|
"loss/base_kto": 3.6570820808410645,
|
|
"metrics/advantage_var": 266.3326721191406,
|
|
"regularization/lipschitz_norm": 1041.120849609375,
|
|
"regularization/w1": 1.0078049898147583,
|
|
"rewards/chosen": 0.26434780638894917,
|
|
"rewards/margins": 0.2948980176797191,
|
|
"rewards/rejected": -0.03055021129076994,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 19.99578285217285,
|
|
"kl": 17.129179000854492,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35817152.40764331,
|
|
"logits/rejected": -38261125.496932514,
|
|
"logps/chosen": -472.1129578025478,
|
|
"logps/rejected": -359.98734662576686,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.671119451522827,
|
|
"metrics/advantage_var": 296.0631408691406,
|
|
"regularization/lipschitz_norm": 1112.0679931640625,
|
|
"regularization/w1": 1.076481819152832,
|
|
"rewards/chosen": 0.25654966512303445,
|
|
"rewards/margins": 0.2995326734316791,
|
|
"rewards/rejected": -0.042983008308644675,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 24.548656463623047,
|
|
"kl": 18.00672721862793,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36440138.635568514,
|
|
"logits/rejected": -37051609.21212121,
|
|
"logps/chosen": -480.29714832361515,
|
|
"logps/rejected": -400.99747474747477,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.6937294006347656,
|
|
"metrics/advantage_var": 275.2530212402344,
|
|
"regularization/lipschitz_norm": 1120.35546875,
|
|
"regularization/w1": 1.0845040082931519,
|
|
"rewards/chosen": 0.33067237253439324,
|
|
"rewards/margins": 0.26438399282667147,
|
|
"rewards/rejected": 0.06628837970772175,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 24.60041046142578,
|
|
"kl": 27.210081100463867,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36826190.76923077,
|
|
"logits/rejected": -37894196.17834395,
|
|
"logps/chosen": -477.15836538461537,
|
|
"logps/rejected": -368.5930284633758,
|
|
"loss": 0.5887,
|
|
"loss/base_kto": 3.494863986968994,
|
|
"metrics/advantage_var": 389.06982421875,
|
|
"regularization/lipschitz_norm": 1254.91796875,
|
|
"regularization/w1": 1.2147605419158936,
|
|
"rewards/chosen": 0.5894831730769231,
|
|
"rewards/margins": 0.5032038203178231,
|
|
"rewards/rejected": 0.08627935275910006,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 16.938255310058594,
|
|
"kl": 18.908203125,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36219343.08280255,
|
|
"logits/rejected": -37101605.693251535,
|
|
"logps/chosen": -465.5640923566879,
|
|
"logps/rejected": -357.50057515337426,
|
|
"loss": 0.6096,
|
|
"loss/base_kto": 3.3356130123138428,
|
|
"metrics/advantage_var": 752.8131713867188,
|
|
"regularization/lipschitz_norm": 1592.1087646484375,
|
|
"regularization/w1": 1.5411611795425415,
|
|
"rewards/chosen": 0.5408358239823845,
|
|
"rewards/margins": 0.7014106375443213,
|
|
"rewards/rejected": -0.16057481356193684,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 17.944320678710938,
|
|
"kl": 23.344526290893555,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -37344155.22539683,
|
|
"logits/rejected": -39009015.335384615,
|
|
"logps/chosen": -479.15396825396823,
|
|
"logps/rejected": -400.1347596153846,
|
|
"loss": 0.6381,
|
|
"loss/base_kto": 3.3081390857696533,
|
|
"metrics/advantage_var": 1236.3426513671875,
|
|
"regularization/lipschitz_norm": 1855.735595703125,
|
|
"regularization/w1": 1.7963520288467407,
|
|
"rewards/chosen": 0.5472180563306052,
|
|
"rewards/margins": 0.7898948470081892,
|
|
"rewards/rejected": -0.24267679067758413,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 16.029644012451172,
|
|
"kl": 18.087770462036133,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -37166595.18012422,
|
|
"logits/rejected": -37989562.7672956,
|
|
"logps/chosen": -484.04886451863354,
|
|
"logps/rejected": -398.87765330188677,
|
|
"loss": 0.611,
|
|
"loss/base_kto": 3.2640769481658936,
|
|
"metrics/advantage_var": 679.4597778320312,
|
|
"regularization/lipschitz_norm": 1677.5914306640625,
|
|
"regularization/w1": 1.6239086389541626,
|
|
"rewards/chosen": 0.5928281227253979,
|
|
"rewards/margins": 0.8127378386777613,
|
|
"rewards/rejected": -0.21990971595236342,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 19.581451416015625,
|
|
"kl": 17.110923767089844,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -37479023.16323297,
|
|
"logits/rejected": -37503777.52850539,
|
|
"logps/chosen": -505.67937797147385,
|
|
"logps/rejected": -365.0031779661017,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.262723684310913,
|
|
"metrics/advantage_var": 696.7647705078125,
|
|
"regularization/lipschitz_norm": 1606.3468017578125,
|
|
"regularization/w1": 1.5549436807632446,
|
|
"rewards/chosen": 0.5062276172184529,
|
|
"rewards/margins": 0.8005683886275919,
|
|
"rewards/rejected": -0.29434077140913906,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 18.793392181396484,
|
|
"kl": 23.380680084228516,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35314973.583975345,
|
|
"logits/rejected": -37076699.89223455,
|
|
"logps/chosen": -485.293624807396,
|
|
"logps/rejected": -380.0300366481775,
|
|
"loss": 0.6174,
|
|
"loss/base_kto": 3.2634105682373047,
|
|
"metrics/advantage_var": 822.0134887695312,
|
|
"regularization/lipschitz_norm": 1730.907470703125,
|
|
"regularization/w1": 1.6755183935165405,
|
|
"rewards/chosen": 0.6396275595266756,
|
|
"rewards/margins": 0.7762186411005972,
|
|
"rewards/rejected": -0.1365910815739216,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 22.290462493896484,
|
|
"kl": 13.332841873168945,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -36854078.71802773,
|
|
"logits/rejected": -37558362.87797147,
|
|
"logps/chosen": -480.96162365177196,
|
|
"logps/rejected": -381.0171354992076,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.31767201423645,
|
|
"metrics/advantage_var": 582.38037109375,
|
|
"regularization/lipschitz_norm": 1569.7359619140625,
|
|
"regularization/w1": 1.519504427909851,
|
|
"rewards/chosen": 0.43009852334420745,
|
|
"rewards/margins": 0.7142735679163152,
|
|
"rewards/rejected": -0.28417504457210774,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 35.066078186035156,
|
|
"kl": 22.25457000732422,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35131025.71695179,
|
|
"logits/rejected": -37416439.15855573,
|
|
"logps/chosen": -467.4997569984448,
|
|
"logps/rejected": -367.7021683673469,
|
|
"loss": 0.6077,
|
|
"loss/base_kto": 3.3176205158233643,
|
|
"metrics/advantage_var": 726.6259155273438,
|
|
"regularization/lipschitz_norm": 1594.919921875,
|
|
"regularization/w1": 1.5438824892044067,
|
|
"rewards/chosen": 0.5890290303326692,
|
|
"rewards/margins": 0.6998086533672796,
|
|
"rewards/rejected": -0.11077962303461047,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 22.020753860473633,
|
|
"kl": 18.579063415527344,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36301629.929121725,
|
|
"logits/rejected": -37965830.49128368,
|
|
"logps/chosen": -514.8477465331279,
|
|
"logps/rejected": -378.4779368066561,
|
|
"loss": 0.6339,
|
|
"loss/base_kto": 3.3006584644317627,
|
|
"metrics/advantage_var": 705.412109375,
|
|
"regularization/lipschitz_norm": 1828.9664306640625,
|
|
"regularization/w1": 1.7704393863677979,
|
|
"rewards/chosen": 0.563342689549427,
|
|
"rewards/margins": 0.7672058909023762,
|
|
"rewards/rejected": -0.2038632013529492,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 19.410432815551758,
|
|
"kl": 14.792784690856934,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35112273.15160796,
|
|
"logits/rejected": -36126870.25199362,
|
|
"logps/chosen": -506.843989280245,
|
|
"logps/rejected": -374.09938197767144,
|
|
"loss": 0.6069,
|
|
"loss/base_kto": 3.295436143875122,
|
|
"metrics/advantage_var": 592.3900756835938,
|
|
"regularization/lipschitz_norm": 1611.45166015625,
|
|
"regularization/w1": 1.5598852634429932,
|
|
"rewards/chosen": 0.5206728784815275,
|
|
"rewards/margins": 0.7591421498599915,
|
|
"rewards/rejected": -0.2384692713784639,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 18.661163330078125,
|
|
"kl": 14.410544395446777,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35926886.4,
|
|
"logits/rejected": -37262348.8,
|
|
"logps/chosen": -461.05478515625,
|
|
"logps/rejected": -373.4477294921875,
|
|
"loss": 0.6033,
|
|
"loss/base_kto": 3.2539584636688232,
|
|
"metrics/advantage_var": 671.54638671875,
|
|
"regularization/lipschitz_norm": 1624.6864013671875,
|
|
"regularization/w1": 1.5726964473724365,
|
|
"rewards/chosen": 0.5405881881713868,
|
|
"rewards/margins": 0.8174409151077271,
|
|
"rewards/rejected": -0.27685272693634033,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 16.210304260253906,
|
|
"kl": 11.0684232711792,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35929776.20779221,
|
|
"logits/rejected": -37701989.78313253,
|
|
"logps/chosen": -469.5021814123377,
|
|
"logps/rejected": -368.64662556475906,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.2493488788604736,
|
|
"metrics/advantage_var": 634.1573486328125,
|
|
"regularization/lipschitz_norm": 1646.1790771484375,
|
|
"regularization/w1": 1.5935014486312866,
|
|
"rewards/chosen": 0.4789655115697291,
|
|
"rewards/margins": 0.8265565670146533,
|
|
"rewards/rejected": -0.34759105544492425,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 18.210792541503906,
|
|
"kl": 9.478139877319336,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34488517.5562701,
|
|
"logits/rejected": -36617847.82978723,
|
|
"logps/chosen": -477.3789188102894,
|
|
"logps/rejected": -368.1649411094225,
|
|
"loss": 0.6017,
|
|
"loss/base_kto": 3.256021499633789,
|
|
"metrics/advantage_var": 621.519775390625,
|
|
"regularization/lipschitz_norm": 1608.736328125,
|
|
"regularization/w1": 1.5572566986083984,
|
|
"rewards/chosen": 0.37491359894681975,
|
|
"rewards/margins": 0.829181338471777,
|
|
"rewards/rejected": -0.45426773952495725,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 19.17514419555664,
|
|
"kl": 9.207308769226074,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -36889585.66718507,
|
|
"logits/rejected": -37244850.83830455,
|
|
"logps/chosen": -493.2518468118196,
|
|
"logps/rejected": -399.2086195054945,
|
|
"loss": 0.6088,
|
|
"loss/base_kto": 3.2761566638946533,
|
|
"metrics/advantage_var": 655.4650268554688,
|
|
"regularization/lipschitz_norm": 1647.1942138671875,
|
|
"regularization/w1": 1.5944842100143433,
|
|
"rewards/chosen": 0.3975367331022793,
|
|
"rewards/margins": 0.7941682823565964,
|
|
"rewards/rejected": -0.3966315492543171,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 17.284456253051758,
|
|
"kl": 11.348340034484863,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35490860.383900926,
|
|
"logits/rejected": -37440363.406940065,
|
|
"logps/chosen": -486.63501354489165,
|
|
"logps/rejected": -378.0798501577287,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.239595890045166,
|
|
"metrics/advantage_var": 652.6485595703125,
|
|
"regularization/lipschitz_norm": 1649.8328857421875,
|
|
"regularization/w1": 1.5970381498336792,
|
|
"rewards/chosen": 0.5580176008005998,
|
|
"rewards/margins": 0.8570709493779608,
|
|
"rewards/rejected": -0.299053348577361,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 20.144357681274414,
|
|
"kl": 9.013777732849121,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35989155.74414977,
|
|
"logits/rejected": -35944630.68544601,
|
|
"logps/chosen": -474.8608131825273,
|
|
"logps/rejected": -378.6284966744914,
|
|
"loss": 0.6017,
|
|
"loss/base_kto": 3.2880914211273193,
|
|
"metrics/advantage_var": 587.4698486328125,
|
|
"regularization/lipschitz_norm": 1575.5634765625,
|
|
"regularization/w1": 1.525145411491394,
|
|
"rewards/chosen": 0.4039212374159029,
|
|
"rewards/margins": 0.7982711490056672,
|
|
"rewards/rejected": -0.3943499115897643,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 22.06709861755371,
|
|
"kl": 11.701579093933105,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34325960.33078101,
|
|
"logits/rejected": -36634088.31897926,
|
|
"logps/chosen": -459.4208460949464,
|
|
"logps/rejected": -376.2984948165869,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.320592164993286,
|
|
"metrics/advantage_var": 557.9876708984375,
|
|
"regularization/lipschitz_norm": 1572.8558349609375,
|
|
"regularization/w1": 1.5225244760513306,
|
|
"rewards/chosen": 0.43199889108561446,
|
|
"rewards/margins": 0.7623338845362599,
|
|
"rewards/rejected": -0.33033499345064543,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 21.966556549072266,
|
|
"kl": 20.693016052246094,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -36104676.36825397,
|
|
"logits/rejected": -37109047.926153846,
|
|
"logps/chosen": -470.13382936507935,
|
|
"logps/rejected": -366.26045673076925,
|
|
"loss": 0.6132,
|
|
"loss/base_kto": 3.2923858165740967,
|
|
"metrics/advantage_var": 639.4615478515625,
|
|
"regularization/lipschitz_norm": 1666.7152099609375,
|
|
"regularization/w1": 1.6133800745010376,
|
|
"rewards/chosen": 0.6354437934027778,
|
|
"rewards/margins": 0.7274205343132345,
|
|
"rewards/rejected": -0.09197674091045673,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 16.00029182434082,
|
|
"kl": 15.745071411132812,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34457289.50689127,
|
|
"logits/rejected": -35155014.226475276,
|
|
"logps/chosen": -469.93472434915776,
|
|
"logps/rejected": -373.8033293460925,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.2756526470184326,
|
|
"metrics/advantage_var": 597.85888671875,
|
|
"regularization/lipschitz_norm": 1575.210693359375,
|
|
"regularization/w1": 1.524803876876831,
|
|
"rewards/chosen": 0.5094848445874809,
|
|
"rewards/margins": 0.7985893001733562,
|
|
"rewards/rejected": -0.2891044555858752,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 23.175050735473633,
|
|
"kl": 18.354515075683594,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -36162743.54716981,
|
|
"logits/rejected": -36677326.70807453,
|
|
"logps/chosen": -485.10254520440253,
|
|
"logps/rejected": -376.70472631987576,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.255446672439575,
|
|
"metrics/advantage_var": 724.1740112304688,
|
|
"regularization/lipschitz_norm": 1640.4986572265625,
|
|
"regularization/w1": 1.5880028009414673,
|
|
"rewards/chosen": 0.6048310957614731,
|
|
"rewards/margins": 0.8083806407652337,
|
|
"rewards/rejected": -0.20354954500376068,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 23.43621253967285,
|
|
"kl": 14.010614395141602,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35727571.40645161,
|
|
"logits/rejected": -37075397.04242424,
|
|
"logps/chosen": -497.9234375,
|
|
"logps/rejected": -383.33451704545456,
|
|
"loss": 0.6041,
|
|
"loss/base_kto": 3.287888765335083,
|
|
"metrics/advantage_var": 582.121337890625,
|
|
"regularization/lipschitz_norm": 1595.81494140625,
|
|
"regularization/w1": 1.5447489023208618,
|
|
"rewards/chosen": 0.4218530470325101,
|
|
"rewards/margins": 0.7655395612222591,
|
|
"rewards/rejected": -0.34368651418974905,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 15.807668685913086,
|
|
"kl": 7.742376804351807,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33806577.314728685,
|
|
"logits/rejected": -35983540.61102362,
|
|
"logps/chosen": -482.26027131782945,
|
|
"logps/rejected": -356.9523622047244,
|
|
"loss": 0.6039,
|
|
"loss/base_kto": 3.2929611206054688,
|
|
"metrics/advantage_var": 607.9397583007812,
|
|
"regularization/lipschitz_norm": 1589.06689453125,
|
|
"regularization/w1": 1.5382165908813477,
|
|
"rewards/chosen": 0.3665622681610344,
|
|
"rewards/margins": 0.7930969899560639,
|
|
"rewards/rejected": -0.42653472179502955,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 16.9594669342041,
|
|
"kl": 9.989230155944824,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34965485.03703704,
|
|
"logits/rejected": -36756424.911392406,
|
|
"logps/chosen": -500.9250578703704,
|
|
"logps/rejected": -371.8223150712025,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.257985830307007,
|
|
"metrics/advantage_var": 582.0637817382812,
|
|
"regularization/lipschitz_norm": 1557.8319091796875,
|
|
"regularization/w1": 1.5079811811447144,
|
|
"rewards/chosen": 0.42141601185739774,
|
|
"rewards/margins": 0.8288047231348403,
|
|
"rewards/rejected": -0.40738871127744264,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 15.593432426452637,
|
|
"kl": 14.027438163757324,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -33736448.394453004,
|
|
"logits/rejected": -35185185.267828844,
|
|
"logps/chosen": -460.82516371340523,
|
|
"logps/rejected": -354.63891640253564,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.351245880126953,
|
|
"metrics/advantage_var": 518.572265625,
|
|
"regularization/lipschitz_norm": 1431.0643310546875,
|
|
"regularization/w1": 1.385270118713379,
|
|
"rewards/chosen": 0.49658184316015025,
|
|
"rewards/margins": 0.7177681228439359,
|
|
"rewards/rejected": -0.22118627968378565,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 19.6863956451416,
|
|
"kl": 7.095364570617676,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35451102.315789476,
|
|
"logits/rejected": -37267516.952380955,
|
|
"logps/chosen": -505.31034128289474,
|
|
"logps/rejected": -380.65478515625,
|
|
"loss": 0.6067,
|
|
"loss/base_kto": 3.3392226696014404,
|
|
"metrics/advantage_var": 636.7014770507812,
|
|
"regularization/lipschitz_norm": 1564.7259521484375,
|
|
"regularization/w1": 1.5146546363830566,
|
|
"rewards/chosen": 0.23330618205823397,
|
|
"rewards/margins": 0.7166173810648141,
|
|
"rewards/rejected": -0.48331119900658015,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 18.977813720703125,
|
|
"kl": 8.8939847946167,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34091576.01232666,
|
|
"logits/rejected": -34983978.1933439,
|
|
"logps/chosen": -491.04853620955316,
|
|
"logps/rejected": -366.8850782488114,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.280792236328125,
|
|
"metrics/advantage_var": 588.0580444335938,
|
|
"regularization/lipschitz_norm": 1594.7991943359375,
|
|
"regularization/w1": 1.543765664100647,
|
|
"rewards/chosen": 0.3843008784924523,
|
|
"rewards/margins": 0.7820609946585082,
|
|
"rewards/rejected": -0.3977601161660559,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 16.42539405822754,
|
|
"kl": 11.247522354125977,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34507682.62132921,
|
|
"logits/rejected": -35061410.57819905,
|
|
"logps/chosen": -471.97082689335394,
|
|
"logps/rejected": -361.99782780410743,
|
|
"loss": 0.5905,
|
|
"loss/base_kto": 3.3281288146972656,
|
|
"metrics/advantage_var": 534.7228393554688,
|
|
"regularization/lipschitz_norm": 1442.419189453125,
|
|
"regularization/w1": 1.3962616920471191,
|
|
"rewards/chosen": 0.3830385753507776,
|
|
"rewards/margins": 0.7063738101268933,
|
|
"rewards/rejected": -0.3233352347761157,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 17.589021682739258,
|
|
"kl": 16.370065689086914,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -35202145.45241581,
|
|
"logits/rejected": -34974996.15410385,
|
|
"logps/chosen": -471.52790995607614,
|
|
"logps/rejected": -361.38929020100505,
|
|
"loss": 0.5971,
|
|
"loss/base_kto": 3.312945604324341,
|
|
"metrics/advantage_var": 549.9900512695312,
|
|
"regularization/lipschitz_norm": 1512.259765625,
|
|
"regularization/w1": 1.4638675451278687,
|
|
"rewards/chosen": 0.5506746395440612,
|
|
"rewards/margins": 0.744134973031158,
|
|
"rewards/rejected": -0.19346033348709693,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 17.559011459350586,
|
|
"kl": 22.12919044494629,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35685087.65905631,
|
|
"logits/rejected": -38136478.613162115,
|
|
"logps/chosen": -465.64911529680364,
|
|
"logps/rejected": -363.81691412520064,
|
|
"loss": 0.5976,
|
|
"loss/base_kto": 3.2522332668304443,
|
|
"metrics/advantage_var": 558.3661499023438,
|
|
"regularization/lipschitz_norm": 1578.8695068359375,
|
|
"regularization/w1": 1.5283458232879639,
|
|
"rewards/chosen": 0.6241924222020072,
|
|
"rewards/margins": 0.7857544401908214,
|
|
"rewards/rejected": -0.1615620179888142,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 19.54766273498535,
|
|
"kl": 21.734264373779297,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35393856.95522388,
|
|
"logits/rejected": -36368172.4852459,
|
|
"logps/chosen": -464.09682835820894,
|
|
"logps/rejected": -394.5155481557377,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.2263848781585693,
|
|
"metrics/advantage_var": 635.239501953125,
|
|
"regularization/lipschitz_norm": 1578.0784912109375,
|
|
"regularization/w1": 1.5275800228118896,
|
|
"rewards/chosen": 0.711181002944263,
|
|
"rewards/margins": 0.8226355007349905,
|
|
"rewards/rejected": -0.11145449779072746,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 18.16437339782715,
|
|
"kl": 24.98301887512207,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35300358.522292994,
|
|
"logits/rejected": -35844661.39877301,
|
|
"logps/chosen": -453.79861664012736,
|
|
"logps/rejected": -390.69854773773005,
|
|
"loss": 0.6004,
|
|
"loss/base_kto": 3.289254903793335,
|
|
"metrics/advantage_var": 638.4701538085938,
|
|
"regularization/lipschitz_norm": 1564.15869140625,
|
|
"regularization/w1": 1.5141055583953857,
|
|
"rewards/chosen": 0.6463654147591561,
|
|
"rewards/margins": 0.7748927927712224,
|
|
"rewards/rejected": -0.12852737801206623,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 22.700044631958008,
|
|
"kl": 19.944427490234375,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34059789.24809741,
|
|
"logits/rejected": -34717962.27287319,
|
|
"logps/chosen": -499.2607496194825,
|
|
"logps/rejected": -377.0922702648475,
|
|
"loss": 0.6041,
|
|
"loss/base_kto": 3.304611921310425,
|
|
"metrics/advantage_var": 623.9252319335938,
|
|
"regularization/lipschitz_norm": 1578.9453125,
|
|
"regularization/w1": 1.5284188985824585,
|
|
"rewards/chosen": 0.53315291266826,
|
|
"rewards/margins": 0.7623388467197247,
|
|
"rewards/rejected": -0.22918593405146467,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 17.395090103149414,
|
|
"kl": 23.02911376953125,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33334811.20772947,
|
|
"logits/rejected": -34795140.85584219,
|
|
"logps/chosen": -478.3526570048309,
|
|
"logps/rejected": -366.0024184370258,
|
|
"loss": 0.5876,
|
|
"loss/base_kto": 3.234140157699585,
|
|
"metrics/advantage_var": 517.2286987304688,
|
|
"regularization/lipschitz_norm": 1515.255859375,
|
|
"regularization/w1": 1.46676766872406,
|
|
"rewards/chosen": 0.5985155581660124,
|
|
"rewards/margins": 0.8077754249296932,
|
|
"rewards/rejected": -0.20925986676368077,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 26.389938354492188,
|
|
"kl": 10.615826606750488,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35018910.75968992,
|
|
"logits/rejected": -37144901.74488189,
|
|
"logps/chosen": -504.83958333333334,
|
|
"logps/rejected": -388.6386811023622,
|
|
"loss": 0.6101,
|
|
"loss/base_kto": 3.2761216163635254,
|
|
"metrics/advantage_var": 616.2353515625,
|
|
"regularization/lipschitz_norm": 1657.9051513671875,
|
|
"regularization/w1": 1.6048519611358643,
|
|
"rewards/chosen": 0.4090720952943314,
|
|
"rewards/margins": 0.7983414419970873,
|
|
"rewards/rejected": -0.3892693467027559,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 19.820838928222656,
|
|
"kl": 10.145833969116211,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -32566207.273905996,
|
|
"logits/rejected": -35092791.987933636,
|
|
"logps/chosen": -470.42767423014584,
|
|
"logps/rejected": -366.9151819381599,
|
|
"loss": 0.6071,
|
|
"loss/base_kto": 3.2971878051757812,
|
|
"metrics/advantage_var": 623.0327758789062,
|
|
"regularization/lipschitz_norm": 1611.2181396484375,
|
|
"regularization/w1": 1.5596592426300049,
|
|
"rewards/chosen": 0.34800089584177474,
|
|
"rewards/margins": 0.7337995155450647,
|
|
"rewards/rejected": -0.38579861970329,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 15.028595924377441,
|
|
"kl": 8.88577938079834,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -33933315.22519685,
|
|
"logits/rejected": -35869935.727131784,
|
|
"logps/chosen": -456.05807086614175,
|
|
"logps/rejected": -364.5686046511628,
|
|
"loss": 0.6036,
|
|
"loss/base_kto": 3.3597493171691895,
|
|
"metrics/advantage_var": 592.4000854492188,
|
|
"regularization/lipschitz_norm": 1517.7269287109375,
|
|
"regularization/w1": 1.469159722328186,
|
|
"rewards/chosen": 0.3310019665815699,
|
|
"rewards/margins": 0.6905269048840166,
|
|
"rewards/rejected": -0.3595249383024467,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 25.656591415405273,
|
|
"kl": 8.698946952819824,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35895312.15141956,
|
|
"logits/rejected": -36742403.9628483,
|
|
"logps/chosen": -510.95376577287067,
|
|
"logps/rejected": -374.7963186919505,
|
|
"loss": 0.6123,
|
|
"loss/base_kto": 3.2607250213623047,
|
|
"metrics/advantage_var": 669.2326049804688,
|
|
"regularization/lipschitz_norm": 1692.150634765625,
|
|
"regularization/w1": 1.638001799583435,
|
|
"rewards/chosen": 0.36136526113804956,
|
|
"rewards/margins": 0.8262746286551016,
|
|
"rewards/rejected": -0.46490936751705203,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 16.233800888061523,
|
|
"kl": 7.031862735748291,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35379684.19169329,
|
|
"logits/rejected": -36600916.550458714,
|
|
"logps/chosen": -495.12974241214056,
|
|
"logps/rejected": -366.6846330275229,
|
|
"loss": 0.5964,
|
|
"loss/base_kto": 3.250248432159424,
|
|
"metrics/advantage_var": 573.9674682617188,
|
|
"regularization/lipschitz_norm": 1570.872314453125,
|
|
"regularization/w1": 1.5206043720245361,
|
|
"rewards/chosen": 0.36616762301411493,
|
|
"rewards/margins": 0.8151972013601108,
|
|
"rewards/rejected": -0.4490295783459958,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 16.594348907470703,
|
|
"kl": 11.1376953125,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -36153678.164781906,
|
|
"logits/rejected": -35587748.986384265,
|
|
"logps/chosen": -485.65165589660745,
|
|
"logps/rejected": -403.3835098335855,
|
|
"loss": 0.596,
|
|
"loss/base_kto": 3.2773282527923584,
|
|
"metrics/advantage_var": 562.6837158203125,
|
|
"regularization/lipschitz_norm": 1540.0648193359375,
|
|
"regularization/w1": 1.4907827377319336,
|
|
"rewards/chosen": 0.36260552475641156,
|
|
"rewards/margins": 0.7602131867091819,
|
|
"rewards/rejected": -0.3976076619527704,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 15.83418083190918,
|
|
"kl": 6.039666652679443,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35821787.31825038,
|
|
"logits/rejected": -35746797.68455285,
|
|
"logps/chosen": -474.98430429864254,
|
|
"logps/rejected": -405.2249745934959,
|
|
"loss": 0.5791,
|
|
"loss/base_kto": 3.2398109436035156,
|
|
"metrics/advantage_var": 499.1890563964844,
|
|
"regularization/lipschitz_norm": 1438.6861572265625,
|
|
"regularization/w1": 1.3926481008529663,
|
|
"rewards/chosen": 0.35221156239689383,
|
|
"rewards/margins": 0.8535869871618837,
|
|
"rewards/rejected": -0.5013754247649899,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 17.7691593170166,
|
|
"kl": 9.771681785583496,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35251361.17011129,
|
|
"logits/rejected": -35749919.459293395,
|
|
"logps/chosen": -475.6163056438792,
|
|
"logps/rejected": -371.68272369431645,
|
|
"loss": 0.5675,
|
|
"loss/base_kto": 3.214143753051758,
|
|
"metrics/advantage_var": 475.9757995605469,
|
|
"regularization/lipschitz_norm": 1369.6226806640625,
|
|
"regularization/w1": 1.3257946968078613,
|
|
"rewards/chosen": 0.4531557601661864,
|
|
"rewards/margins": 0.8741792681706987,
|
|
"rewards/rejected": -0.42102350800451227,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 18.82271385192871,
|
|
"kl": 8.59410285949707,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34081848.33656958,
|
|
"logits/rejected": -35865541.22054381,
|
|
"logps/chosen": -482.58242313915855,
|
|
"logps/rejected": -367.126416163142,
|
|
"loss": 0.5654,
|
|
"loss/base_kto": 3.2107760906219482,
|
|
"metrics/advantage_var": 482.0572204589844,
|
|
"regularization/lipschitz_norm": 1355.5738525390625,
|
|
"regularization/w1": 1.3121955394744873,
|
|
"rewards/chosen": 0.39357182277444885,
|
|
"rewards/margins": 0.8390071306719187,
|
|
"rewards/rejected": -0.44543530789746977,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 19.849895477294922,
|
|
"kl": 9.018664360046387,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34285942.15384615,
|
|
"logits/rejected": -36010002.73170732,
|
|
"logps/chosen": -470.2884114583333,
|
|
"logps/rejected": -369.0609756097561,
|
|
"loss": 0.5485,
|
|
"loss/base_kto": 3.173323631286621,
|
|
"metrics/advantage_var": 435.83251953125,
|
|
"regularization/lipschitz_norm": 1254.4598388671875,
|
|
"regularization/w1": 1.214316964149475,
|
|
"rewards/chosen": 0.3981900092882988,
|
|
"rewards/margins": 0.8660473236074442,
|
|
"rewards/rejected": -0.4678573143191454,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 15.731232643127441,
|
|
"kl": 8.303898811340332,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34342085.16770186,
|
|
"logits/rejected": -35193572.628930815,
|
|
"logps/chosen": -511.21952639751555,
|
|
"logps/rejected": -377.2206662735849,
|
|
"loss": 0.5586,
|
|
"loss/base_kto": 3.1882429122924805,
|
|
"metrics/advantage_var": 426.9867248535156,
|
|
"regularization/lipschitz_norm": 1323.1031494140625,
|
|
"regularization/w1": 1.280763864517212,
|
|
"rewards/chosen": 0.4298747281850495,
|
|
"rewards/margins": 0.8751807024136266,
|
|
"rewards/rejected": -0.44530597422857704,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 17.07794952392578,
|
|
"kl": 9.34009838104248,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -33649992.97737557,
|
|
"logits/rejected": -35155456.82982172,
|
|
"logps/chosen": -469.31004901960785,
|
|
"logps/rejected": -371.00329213938414,
|
|
"loss": 0.5494,
|
|
"loss/base_kto": 3.13303804397583,
|
|
"metrics/advantage_var": 458.3013610839844,
|
|
"regularization/lipschitz_norm": 1303.64990234375,
|
|
"regularization/w1": 1.2619330883026123,
|
|
"rewards/chosen": 0.5222103067113264,
|
|
"rewards/margins": 0.9262919759051321,
|
|
"rewards/rejected": -0.4040816691938057,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 22.245920181274414,
|
|
"kl": 13.897102355957031,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34265748.23161189,
|
|
"logits/rejected": -36153826.44617785,
|
|
"logps/chosen": -494.4129010172144,
|
|
"logps/rejected": -406.0256922776911,
|
|
"loss": 0.5636,
|
|
"loss/base_kto": 3.182051658630371,
|
|
"metrics/advantage_var": 429.80621337890625,
|
|
"regularization/lipschitz_norm": 1370.5823974609375,
|
|
"regularization/w1": 1.3267236948013306,
|
|
"rewards/chosen": 0.5104604133044797,
|
|
"rewards/margins": 0.8476780566733004,
|
|
"rewards/rejected": -0.3372176433688207,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 16.231678009033203,
|
|
"kl": 10.508328437805176,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35065233.662015505,
|
|
"logits/rejected": -35766594.51968504,
|
|
"logps/chosen": -456.8248546511628,
|
|
"logps/rejected": -402.26281988188975,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.214189291000366,
|
|
"metrics/advantage_var": 450.0910339355469,
|
|
"regularization/lipschitz_norm": 1328.478271484375,
|
|
"regularization/w1": 1.2859668731689453,
|
|
"rewards/chosen": 0.4564623928809351,
|
|
"rewards/margins": 0.8440257348777362,
|
|
"rewards/rejected": -0.3875633419968012,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 21.277387619018555,
|
|
"kl": 14.962058067321777,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34369323.47169811,
|
|
"logits/rejected": -36122439.55279503,
|
|
"logps/chosen": -485.31338443396226,
|
|
"logps/rejected": -368.4550174689441,
|
|
"loss": 0.5603,
|
|
"loss/base_kto": 3.199432373046875,
|
|
"metrics/advantage_var": 463.65826416015625,
|
|
"regularization/lipschitz_norm": 1325.4091796875,
|
|
"regularization/w1": 1.2829961776733398,
|
|
"rewards/chosen": 0.5457102457682291,
|
|
"rewards/margins": 0.8430260446994694,
|
|
"rewards/rejected": -0.29731579893124027,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 19.139062881469727,
|
|
"kl": 11.56676959991455,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -34909598.437795274,
|
|
"logits/rejected": -34662241.24031008,
|
|
"logps/chosen": -477.68622047244094,
|
|
"logps/rejected": -378.0814195736434,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.22849702835083,
|
|
"metrics/advantage_var": 413.6275329589844,
|
|
"regularization/lipschitz_norm": 1264.6768798828125,
|
|
"regularization/w1": 1.2242071628570557,
|
|
"rewards/chosen": 0.4646565595011073,
|
|
"rewards/margins": 0.8174664477641886,
|
|
"rewards/rejected": -0.3528098882630814,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 16.831180572509766,
|
|
"kl": 13.030710220336914,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -36171452.294209704,
|
|
"logits/rejected": -36063810.29641186,
|
|
"logps/chosen": -479.4391627543036,
|
|
"logps/rejected": -372.57259165366617,
|
|
"loss": 0.5501,
|
|
"loss/base_kto": 3.176023483276367,
|
|
"metrics/advantage_var": 402.53717041015625,
|
|
"regularization/lipschitz_norm": 1264.9359130859375,
|
|
"regularization/w1": 1.224457859992981,
|
|
"rewards/chosen": 0.480469609650088,
|
|
"rewards/margins": 0.8339350782248857,
|
|
"rewards/rejected": -0.35346546857479766,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 13.404722213745117,
|
|
"kl": 12.666258811950684,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34708866.91350531,
|
|
"logits/rejected": -36114128.59259259,
|
|
"logps/chosen": -468.5335261760243,
|
|
"logps/rejected": -362.0749798711755,
|
|
"loss": 0.5592,
|
|
"loss/base_kto": 3.14180064201355,
|
|
"metrics/advantage_var": 495.2604675292969,
|
|
"regularization/lipschitz_norm": 1375.7337646484375,
|
|
"regularization/w1": 1.3317102193832397,
|
|
"rewards/chosen": 0.5729461345759437,
|
|
"rewards/margins": 0.9146811389948497,
|
|
"rewards/rejected": -0.341735004418906,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 20.48990821838379,
|
|
"kl": 15.638354301452637,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35800157.99033816,
|
|
"logits/rejected": -37840230.943854325,
|
|
"logps/chosen": -495.79478663446054,
|
|
"logps/rejected": -386.48672230652505,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.2032506465911865,
|
|
"metrics/advantage_var": 433.1747131347656,
|
|
"regularization/lipschitz_norm": 1332.3466796875,
|
|
"regularization/w1": 1.289711594581604,
|
|
"rewards/chosen": 0.4842109618747484,
|
|
"rewards/margins": 0.8089658302388654,
|
|
"rewards/rejected": -0.32475486836411704,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 17.898475646972656,
|
|
"kl": 14.179572105407715,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35221684.327044025,
|
|
"logits/rejected": -36688024.64596273,
|
|
"logps/chosen": -474.54107704402514,
|
|
"logps/rejected": -371.8156055900621,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.1838018894195557,
|
|
"metrics/advantage_var": 434.0838317871094,
|
|
"regularization/lipschitz_norm": 1316.9451904296875,
|
|
"regularization/w1": 1.2748029232025146,
|
|
"rewards/chosen": 0.5214571203075865,
|
|
"rewards/margins": 0.8456194791306164,
|
|
"rewards/rejected": -0.3241623588230299,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 22.876934051513672,
|
|
"kl": 13.392501831054688,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -36512782.467817895,
|
|
"logits/rejected": -36692340.653188184,
|
|
"logps/chosen": -489.60223704866564,
|
|
"logps/rejected": -388.48337869362365,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.1998984813690186,
|
|
"metrics/advantage_var": 417.7103576660156,
|
|
"regularization/lipschitz_norm": 1311.892822265625,
|
|
"regularization/w1": 1.2699121236801147,
|
|
"rewards/chosen": 0.5431162117040326,
|
|
"rewards/margins": 0.8235431625888621,
|
|
"rewards/rejected": -0.28042695088482944,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 21.132272720336914,
|
|
"kl": 15.240811347961426,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34950200.88888889,
|
|
"logits/rejected": -36962990.98734177,
|
|
"logps/chosen": -471.61284722222223,
|
|
"logps/rejected": -345.69998516613924,
|
|
"loss": 0.5474,
|
|
"loss/base_kto": 3.1596689224243164,
|
|
"metrics/advantage_var": 428.4873046875,
|
|
"regularization/lipschitz_norm": 1259.7703857421875,
|
|
"regularization/w1": 1.219457745552063,
|
|
"rewards/chosen": 0.57920112138913,
|
|
"rewards/margins": 0.8518852428228227,
|
|
"rewards/rejected": -0.27268412143369264,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 17.1198787689209,
|
|
"kl": 13.534914016723633,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34486941.30120482,
|
|
"logits/rejected": -36017188.571428575,
|
|
"logps/chosen": -483.5800075301205,
|
|
"logps/rejected": -388.0805093344156,
|
|
"loss": 0.5489,
|
|
"loss/base_kto": 3.151799440383911,
|
|
"metrics/advantage_var": 448.9578552246094,
|
|
"regularization/lipschitz_norm": 1280.0438232421875,
|
|
"regularization/w1": 1.2390823364257812,
|
|
"rewards/chosen": 0.6010429658085467,
|
|
"rewards/margins": 0.8929195233940276,
|
|
"rewards/rejected": -0.2918765575854809,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 14.736841201782227,
|
|
"kl": 16.106191635131836,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34115370.267912775,
|
|
"logits/rejected": -35142498.70846395,
|
|
"logps/chosen": -476.6753796728972,
|
|
"logps/rejected": -360.8729917711599,
|
|
"loss": 0.549,
|
|
"loss/base_kto": 3.187504529953003,
|
|
"metrics/advantage_var": 377.8800964355469,
|
|
"regularization/lipschitz_norm": 1244.1793212890625,
|
|
"regularization/w1": 1.204365611076355,
|
|
"rewards/chosen": 0.5910947805624513,
|
|
"rewards/margins": 0.8345600008553589,
|
|
"rewards/rejected": -0.2434652202929075,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 20.697771072387695,
|
|
"kl": 17.210622787475586,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34732319.38064516,
|
|
"logits/rejected": -36867171.2969697,
|
|
"logps/chosen": -492.89309475806454,
|
|
"logps/rejected": -374.50194128787876,
|
|
"loss": 0.5467,
|
|
"loss/base_kto": 3.142366886138916,
|
|
"metrics/advantage_var": 404.5310363769531,
|
|
"regularization/lipschitz_norm": 1271.627685546875,
|
|
"regularization/w1": 1.2309354543685913,
|
|
"rewards/chosen": 0.5891959405714465,
|
|
"rewards/margins": 0.8710199022339702,
|
|
"rewards/rejected": -0.28182396166252366,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 21.43234634399414,
|
|
"kl": 11.712550163269043,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34549284.96466974,
|
|
"logits/rejected": -36040450.034976155,
|
|
"logps/chosen": -490.8580549155146,
|
|
"logps/rejected": -386.1866057233704,
|
|
"loss": 0.5537,
|
|
"loss/base_kto": 3.170138120651245,
|
|
"metrics/advantage_var": 446.85137939453125,
|
|
"regularization/lipschitz_norm": 1301.0899658203125,
|
|
"regularization/w1": 1.2594550848007202,
|
|
"rewards/chosen": 0.5657700686593942,
|
|
"rewards/margins": 0.8920260004226717,
|
|
"rewards/rejected": -0.32625593176327755,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 16.07794761657715,
|
|
"kl": 16.706302642822266,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -33846892.750387594,
|
|
"logits/rejected": -37582386.79685039,
|
|
"logps/chosen": -493.5751937984496,
|
|
"logps/rejected": -350.96998031496065,
|
|
"loss": 0.55,
|
|
"loss/base_kto": 3.166783094406128,
|
|
"metrics/advantage_var": 447.54473876953125,
|
|
"regularization/lipschitz_norm": 1273.734375,
|
|
"regularization/w1": 1.232974886894226,
|
|
"rewards/chosen": 0.5564543021741764,
|
|
"rewards/margins": 0.8642092753379313,
|
|
"rewards/rejected": -0.30775497316375494,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 12.074604988098145,
|
|
"kl": 15.236764907836914,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -33810746.313408725,
|
|
"logits/rejected": -35966213.80937973,
|
|
"logps/chosen": -461.3431946688207,
|
|
"logps/rejected": -371.98071104387293,
|
|
"loss": 0.5606,
|
|
"loss/base_kto": 3.1811397075653076,
|
|
"metrics/advantage_var": 439.155517578125,
|
|
"regularization/lipschitz_norm": 1346.8819580078125,
|
|
"regularization/w1": 1.3037817478179932,
|
|
"rewards/chosen": 0.5613015811162156,
|
|
"rewards/margins": 0.8630208426665418,
|
|
"rewards/rejected": -0.3017192615503262,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 18.371217727661133,
|
|
"kl": 15.394027709960938,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34476951.44744745,
|
|
"logits/rejected": -35661123.543973945,
|
|
"logps/chosen": -475.1385604354354,
|
|
"logps/rejected": -388.6146681596091,
|
|
"loss": 0.5557,
|
|
"loss/base_kto": 3.1971428394317627,
|
|
"metrics/advantage_var": 433.8072204589844,
|
|
"regularization/lipschitz_norm": 1290.0697021484375,
|
|
"regularization/w1": 1.248787522315979,
|
|
"rewards/chosen": 0.5475443705424174,
|
|
"rewards/margins": 0.8538859829202676,
|
|
"rewards/rejected": -0.3063416123778502,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 15.8383150100708,
|
|
"kl": 14.658831596374512,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35672191.37152209,
|
|
"logits/rejected": -36022287.3064275,
|
|
"logps/chosen": -488.82958265139115,
|
|
"logps/rejected": -368.9296524663677,
|
|
"loss": 0.5541,
|
|
"loss/base_kto": 3.2002296447753906,
|
|
"metrics/advantage_var": 391.3653259277344,
|
|
"regularization/lipschitz_norm": 1273.6976318359375,
|
|
"regularization/w1": 1.2329391241073608,
|
|
"rewards/chosen": 0.5046269842918883,
|
|
"rewards/margins": 0.812520745896779,
|
|
"rewards/rejected": -0.3078937616048907,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 13.704710960388184,
|
|
"kl": 13.95847225189209,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35323608.76479515,
|
|
"logits/rejected": -35829690.74396135,
|
|
"logps/chosen": -484.4259294385432,
|
|
"logps/rejected": -386.21230877616745,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.250934362411499,
|
|
"metrics/advantage_var": 388.4420166015625,
|
|
"regularization/lipschitz_norm": 1228.1624755859375,
|
|
"regularization/w1": 1.1888612508773804,
|
|
"rewards/chosen": 0.47963588407803015,
|
|
"rewards/margins": 0.7794944319091492,
|
|
"rewards/rejected": -0.29985854783111915,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 18.413442611694336,
|
|
"kl": 13.51965618133545,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34276617.128682174,
|
|
"logits/rejected": -35582455.130708665,
|
|
"logps/chosen": -483.1273255813953,
|
|
"logps/rejected": -375.75356791338584,
|
|
"loss": 0.5553,
|
|
"loss/base_kto": 3.1614513397216797,
|
|
"metrics/advantage_var": 474.7120056152344,
|
|
"regularization/lipschitz_norm": 1323.4881591796875,
|
|
"regularization/w1": 1.281136393547058,
|
|
"rewards/chosen": 0.5431026961452277,
|
|
"rewards/margins": 0.8802733465973684,
|
|
"rewards/rejected": -0.33717065045214073,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 20.10672378540039,
|
|
"kl": 13.336404800415039,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -36325848.615384616,
|
|
"logits/rejected": -37384994.63763608,
|
|
"logps/chosen": -486.00618131868134,
|
|
"logps/rejected": -371.1384865863142,
|
|
"loss": 0.5464,
|
|
"loss/base_kto": 3.159743070602417,
|
|
"metrics/advantage_var": 397.9462585449219,
|
|
"regularization/lipschitz_norm": 1251.5521240234375,
|
|
"regularization/w1": 1.2115024328231812,
|
|
"rewards/chosen": 0.5362435764570251,
|
|
"rewards/margins": 0.8583157968989235,
|
|
"rewards/rejected": -0.3220722204418983,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 19.93809700012207,
|
|
"kl": 13.473434448242188,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33975308.76012461,
|
|
"logits/rejected": -36267354.68338558,
|
|
"logps/chosen": -461.1892523364486,
|
|
"logps/rejected": -376.57459835423197,
|
|
"loss": 0.5461,
|
|
"loss/base_kto": 3.165024518966675,
|
|
"metrics/advantage_var": 406.1980895996094,
|
|
"regularization/lipschitz_norm": 1243.7576904296875,
|
|
"regularization/w1": 1.203957438468933,
|
|
"rewards/chosen": 0.5525045899958625,
|
|
"rewards/margins": 0.8673262936764929,
|
|
"rewards/rejected": -0.3148217036806304,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 17.971006393432617,
|
|
"kl": 14.84460735321045,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34544989.58308157,
|
|
"logits/rejected": -35355068.06472492,
|
|
"logps/chosen": -477.26586102719034,
|
|
"logps/rejected": -351.91335457119743,
|
|
"loss": 0.5569,
|
|
"loss/base_kto": 3.1866931915283203,
|
|
"metrics/advantage_var": 458.7696228027344,
|
|
"regularization/lipschitz_norm": 1310.2691650390625,
|
|
"regularization/w1": 1.2683404684066772,
|
|
"rewards/chosen": 0.5865461458972574,
|
|
"rewards/margins": 0.8479903858377156,
|
|
"rewards/rejected": -0.2614442399404581,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 15.190082550048828,
|
|
"kl": 13.62389850616455,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34782534.0416,
|
|
"logits/rejected": -35572115.34656489,
|
|
"logps/chosen": -478.90835,
|
|
"logps/rejected": -385.9565601145038,
|
|
"loss": 0.553,
|
|
"loss/base_kto": 3.169635534286499,
|
|
"metrics/advantage_var": 433.609130859375,
|
|
"regularization/lipschitz_norm": 1296.221435546875,
|
|
"regularization/w1": 1.2547422647476196,
|
|
"rewards/chosen": 0.5103400390625,
|
|
"rewards/margins": 0.8577619559950501,
|
|
"rewards/rejected": -0.3474219169325501,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 15.556286811828613,
|
|
"kl": 15.228964805603027,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -36533980.3648,
|
|
"logits/rejected": -36621179.114503816,
|
|
"logps/chosen": -505.37045,
|
|
"logps/rejected": -385.40047709923664,
|
|
"loss": 0.5629,
|
|
"loss/base_kto": 3.183356523513794,
|
|
"metrics/advantage_var": 450.26611328125,
|
|
"regularization/lipschitz_norm": 1363.2025146484375,
|
|
"regularization/w1": 1.3195799589157104,
|
|
"rewards/chosen": 0.5171740234375,
|
|
"rewards/margins": 0.8478632454675572,
|
|
"rewards/rejected": -0.33068922203005724,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 22.656600952148438,
|
|
"kl": 14.536020278930664,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35282791.80613893,
|
|
"logits/rejected": -36202530.0816944,
|
|
"logps/chosen": -480.06613489499193,
|
|
"logps/rejected": -379.86443362329805,
|
|
"loss": 0.5521,
|
|
"loss/base_kto": 3.167524814605713,
|
|
"metrics/advantage_var": 421.6968688964844,
|
|
"regularization/lipschitz_norm": 1290.5283203125,
|
|
"regularization/w1": 1.2492313385009766,
|
|
"rewards/chosen": 0.5458475584359854,
|
|
"rewards/margins": 0.8586908234788891,
|
|
"rewards/rejected": -0.3128432650429037,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 13.293658256530762,
|
|
"kl": 13.152968406677246,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35326902.29709035,
|
|
"logits/rejected": -35520165.767145135,
|
|
"logps/chosen": -489.29431470137825,
|
|
"logps/rejected": -395.8155153508772,
|
|
"loss": 0.561,
|
|
"loss/base_kto": 3.1237094402313232,
|
|
"metrics/advantage_var": 478.9008483886719,
|
|
"regularization/lipschitz_norm": 1409.654541015625,
|
|
"regularization/w1": 1.364545464515686,
|
|
"rewards/chosen": 0.5644781746506509,
|
|
"rewards/margins": 0.9307927356336303,
|
|
"rewards/rejected": -0.3663145609829795,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 17.09130859375,
|
|
"kl": 15.352551460266113,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34775712.867623605,
|
|
"logits/rejected": -34516794.413476266,
|
|
"logps/chosen": -468.2268740031898,
|
|
"logps/rejected": -367.63981144716695,
|
|
"loss": 0.557,
|
|
"loss/base_kto": 3.2039546966552734,
|
|
"metrics/advantage_var": 425.2598571777344,
|
|
"regularization/lipschitz_norm": 1293.800537109375,
|
|
"regularization/w1": 1.2523988485336304,
|
|
"rewards/chosen": 0.5026689982680422,
|
|
"rewards/margins": 0.8029204201161232,
|
|
"rewards/rejected": -0.30025142184808096,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 31.414342880249023,
|
|
"kl": 11.897889137268066,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -36053581.57575758,
|
|
"logits/rejected": -35861672.464516126,
|
|
"logps/chosen": -484.09214015151514,
|
|
"logps/rejected": -371.2914566532258,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.2517166137695312,
|
|
"metrics/advantage_var": 403.1370544433594,
|
|
"regularization/lipschitz_norm": 1275.420166015625,
|
|
"regularization/w1": 1.2346066236495972,
|
|
"rewards/chosen": 0.48069888028231533,
|
|
"rewards/margins": 0.7566803649723355,
|
|
"rewards/rejected": -0.27598148469002015,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 22.863874435424805,
|
|
"kl": 13.133017539978027,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34687439.69811321,
|
|
"logits/rejected": -34936399.50310559,
|
|
"logps/chosen": -478.41922169811323,
|
|
"logps/rejected": -377.55337732919253,
|
|
"loss": 0.5601,
|
|
"loss/base_kto": 3.151587963104248,
|
|
"metrics/advantage_var": 446.29736328125,
|
|
"regularization/lipschitz_norm": 1373.204345703125,
|
|
"regularization/w1": 1.3292616605758667,
|
|
"rewards/chosen": 0.546680138545966,
|
|
"rewards/margins": 0.89350156218909,
|
|
"rewards/rejected": -0.346821423643124,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 16.82756805419922,
|
|
"kl": 13.170405387878418,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34939188.163009405,
|
|
"logits/rejected": -35219602.741433024,
|
|
"logps/chosen": -485.63033894984324,
|
|
"logps/rejected": -354.188765576324,
|
|
"loss": 0.5493,
|
|
"loss/base_kto": 3.2211785316467285,
|
|
"metrics/advantage_var": 382.6482849121094,
|
|
"regularization/lipschitz_norm": 1211.6209716796875,
|
|
"regularization/w1": 1.1728490591049194,
|
|
"rewards/chosen": 0.5260477574267731,
|
|
"rewards/margins": 0.794145584423098,
|
|
"rewards/rejected": -0.26809782699632495,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 18.05494499206543,
|
|
"kl": 12.567227363586426,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34859688.51653543,
|
|
"logits/rejected": -36060142.53643411,
|
|
"logps/chosen": -486.56023622047246,
|
|
"logps/rejected": -381.6423691860465,
|
|
"loss": 0.5697,
|
|
"loss/base_kto": 3.230449676513672,
|
|
"metrics/advantage_var": 466.7301940917969,
|
|
"regularization/lipschitz_norm": 1371.215087890625,
|
|
"regularization/w1": 1.3273361921310425,
|
|
"rewards/chosen": 0.45714466966043305,
|
|
"rewards/margins": 0.7867403093845731,
|
|
"rewards/rejected": -0.32959563972414,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_loss": 0.5855095205323478,
|
|
"train_runtime": 11062.561,
|
|
"train_samples_per_second": 13.398,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.977114557203087e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|