2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 23.520465850830078,
|
|
"kl": 9.559723854064941,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36573467.817322835,
|
|
"logits/rejected": -37330858.26976744,
|
|
"logps/chosen": -466.2831692913386,
|
|
"logps/rejected": -375.03369670542634,
|
|
"loss": 0.5984,
|
|
"loss/base_kto": 3.912484884262085,
|
|
"metrics/advantage_var": 227.017822265625,
|
|
"regularization/lipschitz_norm": 903.7849731445312,
|
|
"regularization/w1": 0.8748638033866882,
|
|
"rewards/chosen": 0.13156353807824803,
|
|
"rewards/margins": 0.0937374287761117,
|
|
"rewards/rejected": 0.037826109302136326,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 22.86282730102539,
|
|
"kl": 7.839565277099609,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36402464.739599384,
|
|
"logits/rejected": -38150728.2155309,
|
|
"logps/chosen": -484.67883281972263,
|
|
"logps/rejected": -376.7168185419968,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.8758480548858643,
|
|
"metrics/advantage_var": 186.01556396484375,
|
|
"regularization/lipschitz_norm": 919.1058959960938,
|
|
"regularization/w1": 0.8896943926811218,
|
|
"rewards/chosen": 0.11659715649526917,
|
|
"rewards/margins": 0.12181235698898651,
|
|
"rewards/rejected": -0.0052152004937173445,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 25.129207611083984,
|
|
"kl": 13.222018241882324,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35916757.134883724,
|
|
"logits/rejected": -36570610.292913385,
|
|
"logps/chosen": -473.3561046511628,
|
|
"logps/rejected": -357.64625984251967,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.888202428817749,
|
|
"metrics/advantage_var": 187.08375549316406,
|
|
"regularization/lipschitz_norm": 902.8727416992188,
|
|
"regularization/w1": 0.8739808201789856,
|
|
"rewards/chosen": 0.20288876082546028,
|
|
"rewards/margins": 0.11168765142739188,
|
|
"rewards/rejected": 0.0912011093980684,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 24.70734214782715,
|
|
"kl": 17.078567504882812,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -38264248.64931087,
|
|
"logits/rejected": -39204700.68261563,
|
|
"logps/chosen": -490.06632848392036,
|
|
"logps/rejected": -375.7382376395534,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.864337921142578,
|
|
"metrics/advantage_var": 176.25901794433594,
|
|
"regularization/lipschitz_norm": 900.7128295898438,
|
|
"regularization/w1": 0.8718900084495544,
|
|
"rewards/chosen": 0.2573632419930848,
|
|
"rewards/margins": 0.11403805690178323,
|
|
"rewards/rejected": 0.14332518509130157,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 18.63149070739746,
|
|
"kl": 4.849026679992676,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36467352.401337795,
|
|
"logits/rejected": -37565298.862170085,
|
|
"logps/chosen": -508.33549331103677,
|
|
"logps/rejected": -382.69377749266863,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.8731982707977295,
|
|
"metrics/advantage_var": 186.1269989013672,
|
|
"regularization/lipschitz_norm": 888.9694213867188,
|
|
"regularization/w1": 0.860522449016571,
|
|
"rewards/chosen": 0.01587622460712956,
|
|
"rewards/margins": 0.10473586102016681,
|
|
"rewards/rejected": -0.08885963641303725,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 17.52667236328125,
|
|
"kl": 4.544734954833984,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35602774.93583725,
|
|
"logits/rejected": -37395460.7925117,
|
|
"logps/chosen": -456.84301643192487,
|
|
"logps/rejected": -359.94583658346335,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.912463426589966,
|
|
"metrics/advantage_var": 184.28514099121094,
|
|
"regularization/lipschitz_norm": 906.2805786132812,
|
|
"regularization/w1": 0.8772795796394348,
|
|
"rewards/chosen": -0.06465410999662254,
|
|
"rewards/margins": 0.08259446610364803,
|
|
"rewards/rejected": -0.14724857610027056,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 18.236589431762695,
|
|
"kl": 3.47149920463562,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -37444398.111801244,
|
|
"logits/rejected": -37990705.91194969,
|
|
"logps/chosen": -500.3855784161491,
|
|
"logps/rejected": -381.7842472484277,
|
|
"loss": 0.5999,
|
|
"loss/base_kto": 3.8246524333953857,
|
|
"metrics/advantage_var": 219.1423797607422,
|
|
"regularization/lipschitz_norm": 1006.7935791015625,
|
|
"regularization/w1": 0.9745761752128601,
|
|
"rewards/chosen": 0.018957638592453478,
|
|
"rewards/margins": 0.1717127761019734,
|
|
"rewards/rejected": -0.15275513750951994,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 18.212522506713867,
|
|
"kl": 5.603143692016602,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36338828.91743119,
|
|
"logits/rejected": -36979590.95207668,
|
|
"logps/chosen": -474.1273413608563,
|
|
"logps/rejected": -364.8186401757188,
|
|
"loss": 0.5892,
|
|
"loss/base_kto": 3.851217269897461,
|
|
"metrics/advantage_var": 188.04832458496094,
|
|
"regularization/lipschitz_norm": 891.1689453125,
|
|
"regularization/w1": 0.8626514673233032,
|
|
"rewards/chosen": 0.04940203278801128,
|
|
"rewards/margins": 0.1352501006840651,
|
|
"rewards/rejected": -0.08584806789605381,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.124338150024414,
|
|
"kl": 17.296937942504883,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -37139267.53374233,
|
|
"logits/rejected": -37518583.847133756,
|
|
"logps/chosen": -460.98015720858893,
|
|
"logps/rejected": -364.12318371815286,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.7648568153381348,
|
|
"metrics/advantage_var": 239.6884002685547,
|
|
"regularization/lipschitz_norm": 1036.94921875,
|
|
"regularization/w1": 1.0037668943405151,
|
|
"rewards/chosen": 0.36062310809738063,
|
|
"rewards/margins": 0.21777830660159167,
|
|
"rewards/rejected": 0.14284480149578896,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 26.625356674194336,
|
|
"kl": 6.8466796875,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -38605282.06769231,
|
|
"logits/rejected": -38005958.298412696,
|
|
"logps/chosen": -492.83259615384617,
|
|
"logps/rejected": -402.50664682539684,
|
|
"loss": 0.6045,
|
|
"loss/base_kto": 3.838498830795288,
|
|
"metrics/advantage_var": 229.38780212402344,
|
|
"regularization/lipschitz_norm": 1030.2513427734375,
|
|
"regularization/w1": 0.9972832798957825,
|
|
"rewards/chosen": 0.08453700725848858,
|
|
"rewards/margins": 0.14766808534279846,
|
|
"rewards/rejected": -0.0631310780843099,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 18.594072341918945,
|
|
"kl": 6.898728847503662,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37655598.618524335,
|
|
"logits/rejected": -37721694.75583204,
|
|
"logps/chosen": -493.7249803767661,
|
|
"logps/rejected": -379.38340785381024,
|
|
"loss": 0.6029,
|
|
"loss/base_kto": 3.8260109424591064,
|
|
"metrics/advantage_var": 221.1991424560547,
|
|
"regularization/lipschitz_norm": 1030.2867431640625,
|
|
"regularization/w1": 0.9973174929618835,
|
|
"rewards/chosen": 0.07648501553378262,
|
|
"rewards/margins": 0.16796560882207967,
|
|
"rewards/rejected": -0.09148059328829705,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 23.81263542175293,
|
|
"kl": 4.941190242767334,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -38719922.122977346,
|
|
"logits/rejected": -38529834.53776435,
|
|
"logps/chosen": -480.89492313915855,
|
|
"logps/rejected": -381.4599461858006,
|
|
"loss": 0.6019,
|
|
"loss/base_kto": 3.834670305252075,
|
|
"metrics/advantage_var": 222.9567413330078,
|
|
"regularization/lipschitz_norm": 1012.8854370117188,
|
|
"regularization/w1": 0.9804731607437134,
|
|
"rewards/chosen": -0.013743284836556147,
|
|
"rewards/margins": 0.15511639865835747,
|
|
"rewards/rejected": -0.16885968349491362,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 23.887531280517578,
|
|
"kl": 3.0359668731689453,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -37263693.5504886,
|
|
"logits/rejected": -38677746.93093093,
|
|
"logps/chosen": -517.8246131921824,
|
|
"logps/rejected": -372.8805602477477,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.8057708740234375,
|
|
"metrics/advantage_var": 264.290283203125,
|
|
"regularization/lipschitz_norm": 1052.3267822265625,
|
|
"regularization/w1": 1.018652319908142,
|
|
"rewards/chosen": -0.02830757613290793,
|
|
"rewards/margins": 0.18770690789457659,
|
|
"rewards/rejected": -0.21601448402748452,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 22.569393157958984,
|
|
"kl": 2.0139083862304688,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -38007547.4944,
|
|
"logits/rejected": -39162847.169465646,
|
|
"logps/chosen": -517.6425,
|
|
"logps/rejected": -392.12709923664124,
|
|
"loss": 0.5902,
|
|
"loss/base_kto": 3.7578494548797607,
|
|
"metrics/advantage_var": 232.45201110839844,
|
|
"regularization/lipschitz_norm": 995.5601806640625,
|
|
"regularization/w1": 0.9637022018432617,
|
|
"rewards/chosen": -0.10413770751953125,
|
|
"rewards/margins": 0.24090651734330273,
|
|
"rewards/rejected": -0.345044224862834,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 19.67327880859375,
|
|
"kl": 3.7365338802337646,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36432668.8,
|
|
"logits/rejected": -37197299.2,
|
|
"logps/chosen": -499.42978515625,
|
|
"logps/rejected": -353.5060302734375,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.7835915088653564,
|
|
"metrics/advantage_var": 232.3800811767578,
|
|
"regularization/lipschitz_norm": 1013.5687866210938,
|
|
"regularization/w1": 0.9811345338821411,
|
|
"rewards/chosen": -0.15913006067276,
|
|
"rewards/margins": 0.2147739291191101,
|
|
"rewards/rejected": -0.3739039897918701,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 19.818883895874023,
|
|
"kl": 8.405693054199219,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35768579.91437309,
|
|
"logits/rejected": -38325876.14057508,
|
|
"logps/chosen": -489.362003058104,
|
|
"logps/rejected": -384.3630940495208,
|
|
"loss": 0.6017,
|
|
"loss/base_kto": 3.8035829067230225,
|
|
"metrics/advantage_var": 238.9305877685547,
|
|
"regularization/lipschitz_norm": 1043.50634765625,
|
|
"regularization/w1": 1.010114073753357,
|
|
"rewards/chosen": 0.0986216833832067,
|
|
"rewards/margins": 0.18303705823656632,
|
|
"rewards/rejected": -0.08441537485335962,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 23.001989364624023,
|
|
"kl": 11.807660102844238,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -38333275.34456355,
|
|
"logits/rejected": -38490030.34130781,
|
|
"logps/chosen": -491.9698028330781,
|
|
"logps/rejected": -378.8022328548644,
|
|
"loss": 0.601,
|
|
"loss/base_kto": 3.831860065460205,
|
|
"metrics/advantage_var": 239.6153564453125,
|
|
"regularization/lipschitz_norm": 1008.1463012695312,
|
|
"regularization/w1": 0.9758855700492859,
|
|
"rewards/chosen": 0.19005291458293452,
|
|
"rewards/margins": 0.15743175602566933,
|
|
"rewards/rejected": 0.03262115855726519,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 20.453725814819336,
|
|
"kl": 6.2604522705078125,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36573856.820512824,
|
|
"logits/rejected": -37495033.75609756,
|
|
"logps/chosen": -495.8428485576923,
|
|
"logps/rejected": -385.2627191310976,
|
|
"loss": 0.5822,
|
|
"loss/base_kto": 3.713939666748047,
|
|
"metrics/advantage_var": 225.63101196289062,
|
|
"regularization/lipschitz_norm": 974.8565673828125,
|
|
"regularization/w1": 0.9436611533164978,
|
|
"rewards/chosen": 0.08053700740520771,
|
|
"rewards/margins": 0.27201938852807594,
|
|
"rewards/rejected": -0.19148238112286822,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 19.93045997619629,
|
|
"kl": 10.168670654296875,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35985685.10108865,
|
|
"logits/rejected": -37069198.66875981,
|
|
"logps/chosen": -483.7433903576983,
|
|
"logps/rejected": -352.7525019623234,
|
|
"loss": 0.5892,
|
|
"loss/base_kto": 3.7523553371429443,
|
|
"metrics/advantage_var": 223.45669555664062,
|
|
"regularization/lipschitz_norm": 992.98681640625,
|
|
"regularization/w1": 0.9612112045288086,
|
|
"rewards/chosen": 0.19000422120279695,
|
|
"rewards/margins": 0.2283782312259769,
|
|
"rewards/rejected": -0.038374010023179944,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 26.847291946411133,
|
|
"kl": 4.63355016708374,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -36002406.4,
|
|
"logits/rejected": -38314900.723809525,
|
|
"logps/chosen": -508.5522596153846,
|
|
"logps/rejected": -365.57745535714287,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.8035857677459717,
|
|
"metrics/advantage_var": 238.40640258789062,
|
|
"regularization/lipschitz_norm": 1047.6689453125,
|
|
"regularization/w1": 1.014143466949463,
|
|
"rewards/chosen": 0.09812793438251202,
|
|
"rewards/margins": 0.19709588317498472,
|
|
"rewards/rejected": -0.09896794879247271,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 22.135398864746094,
|
|
"kl": 11.652030944824219,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -37296924.44444445,
|
|
"logits/rejected": -37519846.07594936,
|
|
"logps/chosen": -510.19434799382714,
|
|
"logps/rejected": -389.2604825949367,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.7391815185546875,
|
|
"metrics/advantage_var": 252.87466430664062,
|
|
"regularization/lipschitz_norm": 1050.7498779296875,
|
|
"regularization/w1": 1.0171257257461548,
|
|
"rewards/chosen": 0.2549274821340302,
|
|
"rewards/margins": 0.23427076365057614,
|
|
"rewards/rejected": 0.020656718483454066,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 24.089962005615234,
|
|
"kl": 5.65649938583374,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37029584.7133758,
|
|
"logits/rejected": -37732355.141104296,
|
|
"logps/chosen": -489.65406050955414,
|
|
"logps/rejected": -378.8366804064417,
|
|
"loss": 0.5924,
|
|
"loss/base_kto": 3.745620012283325,
|
|
"metrics/advantage_var": 240.201171875,
|
|
"regularization/lipschitz_norm": 1026.6197509765625,
|
|
"regularization/w1": 0.9937679171562195,
|
|
"rewards/chosen": 0.0006922259452236686,
|
|
"rewards/margins": 0.24105943666104998,
|
|
"rewards/rejected": -0.2403672107158263,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 14.913313865661621,
|
|
"kl": 5.1705474853515625,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36314223.65109035,
|
|
"logits/rejected": -36978142.29467085,
|
|
"logps/chosen": -476.7765284267913,
|
|
"logps/rejected": -366.6149588557994,
|
|
"loss": 0.5849,
|
|
"loss/base_kto": 3.686143159866333,
|
|
"metrics/advantage_var": 245.3922576904297,
|
|
"regularization/lipschitz_norm": 1025.6868896484375,
|
|
"regularization/w1": 0.9928650259971619,
|
|
"rewards/chosen": 0.08335568674628237,
|
|
"rewards/margins": 0.32021901736013914,
|
|
"rewards/rejected": -0.23686333061385678,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 20.646709442138672,
|
|
"kl": 4.831476211547852,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36356099.22012579,
|
|
"logits/rejected": -36889406.01242236,
|
|
"logps/chosen": -474.0918337264151,
|
|
"logps/rejected": -382.4863887810559,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.8170406818389893,
|
|
"metrics/advantage_var": 217.2059783935547,
|
|
"regularization/lipschitz_norm": 993.1124877929688,
|
|
"regularization/w1": 0.9613329172134399,
|
|
"rewards/chosen": -0.0247116448744288,
|
|
"rewards/margins": 0.18163210664197257,
|
|
"rewards/rejected": -0.20634375151640139,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 19.28455352783203,
|
|
"kl": 11.628500938415527,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -35704644.110091746,
|
|
"logits/rejected": -35700624.766773164,
|
|
"logps/chosen": -472.08471903669727,
|
|
"logps/rejected": -371.8917232428115,
|
|
"loss": 0.5865,
|
|
"loss/base_kto": 3.700636386871338,
|
|
"metrics/advantage_var": 245.6863250732422,
|
|
"regularization/lipschitz_norm": 1023.8523559570312,
|
|
"regularization/w1": 0.9910890460014343,
|
|
"rewards/chosen": 0.2200622091964115,
|
|
"rewards/margins": 0.2778344159995109,
|
|
"rewards/rejected": -0.05777220680309941,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 20.449249267578125,
|
|
"kl": 10.137863159179688,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36370151.67793881,
|
|
"logits/rejected": -36729944.57056145,
|
|
"logps/chosen": -461.4433373590982,
|
|
"logps/rejected": -368.6161798179059,
|
|
"loss": 0.5872,
|
|
"loss/base_kto": 3.7429165840148926,
|
|
"metrics/advantage_var": 231.12254333496094,
|
|
"regularization/lipschitz_norm": 985.87890625,
|
|
"regularization/w1": 0.9543307423591614,
|
|
"rewards/chosen": 0.05665735806819897,
|
|
"rewards/margins": 0.22892186239234066,
|
|
"rewards/rejected": -0.1722645043241417,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 21.569353103637695,
|
|
"kl": 10.577391624450684,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36510891.9346211,
|
|
"logits/rejected": -38183792.6062603,
|
|
"logps/chosen": -502.57958766716195,
|
|
"logps/rejected": -379.09776565074134,
|
|
"loss": 0.5972,
|
|
"loss/base_kto": 3.747912645339966,
|
|
"metrics/advantage_var": 265.0936584472656,
|
|
"regularization/lipschitz_norm": 1063.8216552734375,
|
|
"regularization/w1": 1.029779314994812,
|
|
"rewards/chosen": 0.2105780455545366,
|
|
"rewards/margins": 0.24595524633948693,
|
|
"rewards/rejected": -0.03537720078495032,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 25.24627685546875,
|
|
"kl": 15.648709297180176,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -36354528.0,
|
|
"logits/rejected": -38008864.0,
|
|
"logps/chosen": -486.124755859375,
|
|
"logps/rejected": -382.7644287109375,
|
|
"loss": 0.5956,
|
|
"loss/base_kto": 3.7468719482421875,
|
|
"metrics/advantage_var": 265.0270690917969,
|
|
"regularization/lipschitz_norm": 1051.7054443359375,
|
|
"regularization/w1": 1.0180509090423584,
|
|
"rewards/chosen": 0.270208477973938,
|
|
"rewards/margins": 0.227033793926239,
|
|
"rewards/rejected": 0.043174684047698975,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 18.384124755859375,
|
|
"kl": 6.6732072830200195,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35663425.39563863,
|
|
"logits/rejected": -37243027.661442004,
|
|
"logps/chosen": -473.654984423676,
|
|
"logps/rejected": -356.44915752351096,
|
|
"loss": 0.5889,
|
|
"loss/base_kto": 3.703855276107788,
|
|
"metrics/advantage_var": 313.8638610839844,
|
|
"regularization/lipschitz_norm": 1040.3367919921875,
|
|
"regularization/w1": 1.007045865058899,
|
|
"rewards/chosen": 0.09530495854553032,
|
|
"rewards/margins": 0.28566617282752726,
|
|
"rewards/rejected": -0.19036121428199695,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 20.570220947265625,
|
|
"kl": 15.123217582702637,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36287042.29641186,
|
|
"logits/rejected": -38202443.31768388,
|
|
"logps/chosen": -489.9688475039002,
|
|
"logps/rejected": -355.72486306729263,
|
|
"loss": 0.5968,
|
|
"loss/base_kto": 3.6963062286376953,
|
|
"metrics/advantage_var": 272.5968322753906,
|
|
"regularization/lipschitz_norm": 1114.0543212890625,
|
|
"regularization/w1": 1.0784046649932861,
|
|
"rewards/chosen": 0.3411232268382532,
|
|
"rewards/margins": 0.26919406530083206,
|
|
"rewards/rejected": 0.07192916153742114,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 22.337671279907227,
|
|
"kl": 15.512511253356934,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37349353.842349306,
|
|
"logits/rejected": -37635232.151658766,
|
|
"logps/chosen": -492.96841190108194,
|
|
"logps/rejected": -362.6146327014218,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.794390916824341,
|
|
"metrics/advantage_var": 221.1488800048828,
|
|
"regularization/lipschitz_norm": 991.7467041015625,
|
|
"regularization/w1": 0.9600107073783875,
|
|
"rewards/chosen": 0.20758990563417698,
|
|
"rewards/margins": 0.16674419134832494,
|
|
"rewards/rejected": 0.040845714285852035,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 18.192623138427734,
|
|
"kl": 14.482287406921387,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -36140539.331306994,
|
|
"logits/rejected": -37770302.55948553,
|
|
"logps/chosen": -466.1228628419453,
|
|
"logps/rejected": -376.8180265273312,
|
|
"loss": 0.5935,
|
|
"loss/base_kto": 3.700916290283203,
|
|
"metrics/advantage_var": 262.0703125,
|
|
"regularization/lipschitz_norm": 1081.7369384765625,
|
|
"regularization/w1": 1.0471214056015015,
|
|
"rewards/chosen": 0.27024813481014914,
|
|
"rewards/margins": 0.26756823561898274,
|
|
"rewards/rejected": 0.0026798991911664268,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 24.54085350036621,
|
|
"kl": 6.698326110839844,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36780085.64560863,
|
|
"logits/rejected": -37567236.46275753,
|
|
"logps/chosen": -483.77532742681046,
|
|
"logps/rejected": -385.76371830427894,
|
|
"loss": 0.5981,
|
|
"loss/base_kto": 3.6923840045928955,
|
|
"metrics/advantage_var": 287.8490295410156,
|
|
"regularization/lipschitz_norm": 1128.940673828125,
|
|
"regularization/w1": 1.0928146839141846,
|
|
"rewards/chosen": 0.08990239876628106,
|
|
"rewards/margins": 0.3034870689404218,
|
|
"rewards/rejected": -0.21358467017414076,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 22.14678192138672,
|
|
"kl": 14.191081047058105,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35047067.5443038,
|
|
"logits/rejected": -35799665.777777776,
|
|
"logps/chosen": -472.31853243670884,
|
|
"logps/rejected": -369.9242862654321,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.7043144702911377,
|
|
"metrics/advantage_var": 273.3419494628906,
|
|
"regularization/lipschitz_norm": 1113.583251953125,
|
|
"regularization/w1": 1.0779485702514648,
|
|
"rewards/chosen": 0.22313999224312697,
|
|
"rewards/margins": 0.2560939021288631,
|
|
"rewards/rejected": -0.03295390988573616,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 27.54254913330078,
|
|
"kl": 18.231040954589844,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -35833040.65203762,
|
|
"logits/rejected": -36451209.96884735,
|
|
"logps/chosen": -465.7884012539185,
|
|
"logps/rejected": -353.8194606697819,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.805208683013916,
|
|
"metrics/advantage_var": 219.84739685058594,
|
|
"regularization/lipschitz_norm": 981.9959106445312,
|
|
"regularization/w1": 0.9505720138549805,
|
|
"rewards/chosen": 0.17400730814679663,
|
|
"rewards/margins": 0.15589875602809714,
|
|
"rewards/rejected": 0.018108552118699498,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 17.976943969726562,
|
|
"kl": 8.278478622436523,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -37288787.451968506,
|
|
"logits/rejected": -37389310.4124031,
|
|
"logps/chosen": -494.08149606299213,
|
|
"logps/rejected": -368.29990310077517,
|
|
"loss": 0.5941,
|
|
"loss/base_kto": 3.7245163917541504,
|
|
"metrics/advantage_var": 255.7454376220703,
|
|
"regularization/lipschitz_norm": 1062.4549560546875,
|
|
"regularization/w1": 1.0284563302993774,
|
|
"rewards/chosen": 0.09054472915769562,
|
|
"rewards/margins": 0.25906997547476807,
|
|
"rewards/rejected": -0.16852524631707244,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 23.18642807006836,
|
|
"kl": 13.88806438446045,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35624080.92068429,
|
|
"logits/rejected": -37473818.52433281,
|
|
"logps/chosen": -476.29471228615864,
|
|
"logps/rejected": -388.41098410518055,
|
|
"loss": 0.5915,
|
|
"loss/base_kto": 3.7031381130218506,
|
|
"metrics/advantage_var": 245.09634399414062,
|
|
"regularization/lipschitz_norm": 1062.5111083984375,
|
|
"regularization/w1": 1.0285106897354126,
|
|
"rewards/chosen": 0.17167295749599049,
|
|
"rewards/margins": 0.26777254751604296,
|
|
"rewards/rejected": -0.09609959002005249,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 21.30337905883789,
|
|
"kl": 15.486564636230469,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35362729.977382876,
|
|
"logits/rejected": -35705144.93192133,
|
|
"logps/chosen": -475.74126615508885,
|
|
"logps/rejected": -360.28399205748866,
|
|
"loss": 0.5843,
|
|
"loss/base_kto": 3.7010204792022705,
|
|
"metrics/advantage_var": 247.0442352294922,
|
|
"regularization/lipschitz_norm": 1005.2230834960938,
|
|
"regularization/w1": 0.9730558395385742,
|
|
"rewards/chosen": 0.2847512141953377,
|
|
"rewards/margins": 0.2657792011229184,
|
|
"rewards/rejected": 0.018972013072419275,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 22.381240844726562,
|
|
"kl": 7.898492336273193,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -37080220.686244205,
|
|
"logits/rejected": -37749316.96988907,
|
|
"logps/chosen": -505.8435568006182,
|
|
"logps/rejected": -376.1768522187005,
|
|
"loss": 0.5987,
|
|
"loss/base_kto": 3.559502363204956,
|
|
"metrics/advantage_var": 378.0544738769531,
|
|
"regularization/lipschitz_norm": 1271.1656494140625,
|
|
"regularization/w1": 1.2304881811141968,
|
|
"rewards/chosen": 0.19574752878368915,
|
|
"rewards/margins": 0.4762175410797098,
|
|
"rewards/rejected": -0.2804700122960207,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 21.085912704467773,
|
|
"kl": 10.179356575012207,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36157222.4,
|
|
"logits/rejected": -37276028.8,
|
|
"logps/chosen": -477.5181640625,
|
|
"logps/rejected": -373.72646484375,
|
|
"loss": 0.6028,
|
|
"loss/base_kto": 3.2891101837158203,
|
|
"metrics/advantage_var": 580.4597778320312,
|
|
"regularization/lipschitz_norm": 1584.220458984375,
|
|
"regularization/w1": 1.5335255861282349,
|
|
"rewards/chosen": 0.4128760814666748,
|
|
"rewards/margins": 0.7816151142120362,
|
|
"rewards/rejected": -0.36873903274536135,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 23.974157333374023,
|
|
"kl": 19.186811447143555,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36094889.27191679,
|
|
"logits/rejected": -37442683.149917625,
|
|
"logps/chosen": -483.99832838038634,
|
|
"logps/rejected": -373.0881383855025,
|
|
"loss": 0.603,
|
|
"loss/base_kto": 3.274304151535034,
|
|
"metrics/advantage_var": 717.5619506835938,
|
|
"regularization/lipschitz_norm": 1600.6436767578125,
|
|
"regularization/w1": 1.549423098564148,
|
|
"rewards/chosen": 0.5647321363792023,
|
|
"rewards/margins": 0.8161965276445184,
|
|
"rewards/rejected": -0.2514643912653161,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 23.855680465698242,
|
|
"kl": 14.738944053649902,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -37324991.18987342,
|
|
"logits/rejected": -37315941.13580247,
|
|
"logps/chosen": -474.4146558544304,
|
|
"logps/rejected": -364.94294945987656,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.2867438793182373,
|
|
"metrics/advantage_var": 546.3062133789062,
|
|
"regularization/lipschitz_norm": 1534.7503662109375,
|
|
"regularization/w1": 1.4856384992599487,
|
|
"rewards/chosen": 0.4704218997231013,
|
|
"rewards/margins": 0.7818954871359943,
|
|
"rewards/rejected": -0.31147358741289305,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 18.959775924682617,
|
|
"kl": 18.78352928161621,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35655975.68389058,
|
|
"logits/rejected": -37555624.74598071,
|
|
"logps/chosen": -488.68313069908817,
|
|
"logps/rejected": -372.5703878617363,
|
|
"loss": 0.6067,
|
|
"loss/base_kto": 3.297025442123413,
|
|
"metrics/advantage_var": 640.2606811523438,
|
|
"regularization/lipschitz_norm": 1607.813720703125,
|
|
"regularization/w1": 1.5563637018203735,
|
|
"rewards/chosen": 0.56435642416354,
|
|
"rewards/margins": 0.7751931965302754,
|
|
"rewards/rejected": -0.21083677236673534,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 20.264001846313477,
|
|
"kl": 10.69848918914795,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34956656.64,
|
|
"logits/rejected": -37117061.28253968,
|
|
"logps/chosen": -491.86721153846156,
|
|
"logps/rejected": -360.2543650793651,
|
|
"loss": 0.6012,
|
|
"loss/base_kto": 3.3239266872406006,
|
|
"metrics/advantage_var": 590.2598876953125,
|
|
"regularization/lipschitz_norm": 1534.8402099609375,
|
|
"regularization/w1": 1.4857252836227417,
|
|
"rewards/chosen": 0.4336868990384615,
|
|
"rewards/margins": 0.7405635266426283,
|
|
"rewards/rejected": -0.3068766276041667,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 14.950648307800293,
|
|
"kl": 18.850784301757812,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35931120.24615385,
|
|
"logits/rejected": -37430109.46031746,
|
|
"logps/chosen": -488.4308653846154,
|
|
"logps/rejected": -357.2547123015873,
|
|
"loss": 0.5925,
|
|
"loss/base_kto": 3.2882537841796875,
|
|
"metrics/advantage_var": 573.1253051757812,
|
|
"regularization/lipschitz_norm": 1499.5260009765625,
|
|
"regularization/w1": 1.4515411853790283,
|
|
"rewards/chosen": 0.5488313645582933,
|
|
"rewards/margins": 0.7731136444054535,
|
|
"rewards/rejected": -0.22428227984716023,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 19.471118927001953,
|
|
"kl": 23.11271858215332,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -36322602.04227642,
|
|
"logits/rejected": -38412081.66015038,
|
|
"logps/chosen": -472.40985772357726,
|
|
"logps/rejected": -393.04290413533835,
|
|
"loss": 0.6127,
|
|
"loss/base_kto": 3.2208099365234375,
|
|
"metrics/advantage_var": 713.4490356445312,
|
|
"regularization/lipschitz_norm": 1736.012939453125,
|
|
"regularization/w1": 1.6804603338241577,
|
|
"rewards/chosen": 0.616719911156631,
|
|
"rewards/margins": 0.8268208623872466,
|
|
"rewards/rejected": -0.2101009512306156,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 20.999032974243164,
|
|
"kl": 8.258482933044434,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35769638.76206323,
|
|
"logits/rejected": -36273705.472754054,
|
|
"logps/chosen": -493.8125519966722,
|
|
"logps/rejected": -387.9956737849779,
|
|
"loss": 0.612,
|
|
"loss/base_kto": 3.3145501613616943,
|
|
"metrics/advantage_var": 600.8352661132812,
|
|
"regularization/lipschitz_norm": 1633.7086181640625,
|
|
"regularization/w1": 1.5814298391342163,
|
|
"rewards/chosen": 0.21964414980566244,
|
|
"rewards/margins": 0.7550540238017736,
|
|
"rewards/rejected": -0.5354098739961111,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 21.01276206970215,
|
|
"kl": 5.731344699859619,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -36356501.73584906,
|
|
"logits/rejected": -37665353.14285714,
|
|
"logps/chosen": -496.8933274371069,
|
|
"logps/rejected": -370.7128299689441,
|
|
"loss": 0.6035,
|
|
"loss/base_kto": 3.2704904079437256,
|
|
"metrics/advantage_var": 625.6759643554688,
|
|
"regularization/lipschitz_norm": 1608.796875,
|
|
"regularization/w1": 1.5573153495788574,
|
|
"rewards/chosen": 0.25753153195171236,
|
|
"rewards/margins": 0.8548342234883728,
|
|
"rewards/rejected": -0.5973026915366605,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 19.614688873291016,
|
|
"kl": 7.569427490234375,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35500696.935064934,
|
|
"logits/rejected": -37502994.5060241,
|
|
"logps/chosen": -503.39782873376623,
|
|
"logps/rejected": -403.2844503012048,
|
|
"loss": 0.6104,
|
|
"loss/base_kto": 3.341423749923706,
|
|
"metrics/advantage_var": 631.2008666992188,
|
|
"regularization/lipschitz_norm": 1592.6756591796875,
|
|
"regularization/w1": 1.5417098999023438,
|
|
"rewards/chosen": 0.2732985360281808,
|
|
"rewards/margins": 0.7208249852086919,
|
|
"rewards/rejected": -0.4475264491805111,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 25.39152717590332,
|
|
"kl": 10.260856628417969,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34952631.754601225,
|
|
"logits/rejected": -35534626.24203822,
|
|
"logps/chosen": -480.83981978527606,
|
|
"logps/rejected": -383.83076234076435,
|
|
"loss": 0.6062,
|
|
"loss/base_kto": 3.270432233810425,
|
|
"metrics/advantage_var": 657.7363891601562,
|
|
"regularization/lipschitz_norm": 1631.656005859375,
|
|
"regularization/w1": 1.5794429779052734,
|
|
"rewards/chosen": 0.463014146301644,
|
|
"rewards/margins": 0.8106554290817993,
|
|
"rewards/rejected": -0.34764128278015527,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 18.431480407714844,
|
|
"kl": 13.851654052734375,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35060143.32121212,
|
|
"logits/rejected": -37286396.69677419,
|
|
"logps/chosen": -492.8298768939394,
|
|
"logps/rejected": -362.8032762096774,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.3003945350646973,
|
|
"metrics/advantage_var": 589.3839111328125,
|
|
"regularization/lipschitz_norm": 1601.48193359375,
|
|
"regularization/w1": 1.5502346754074097,
|
|
"rewards/chosen": 0.4523438424775095,
|
|
"rewards/margins": 0.7649832881207806,
|
|
"rewards/rejected": -0.3126394456432712,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 23.38780975341797,
|
|
"kl": 11.687821388244629,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35362228.589147285,
|
|
"logits/rejected": -37175386.30551181,
|
|
"logps/chosen": -491.5002906976744,
|
|
"logps/rejected": -369.6359251968504,
|
|
"loss": 0.6046,
|
|
"loss/base_kto": 3.2948410511016846,
|
|
"metrics/advantage_var": 624.1443481445312,
|
|
"regularization/lipschitz_norm": 1593.1685791015625,
|
|
"regularization/w1": 1.5421870946884155,
|
|
"rewards/chosen": 0.4840774890988372,
|
|
"rewards/margins": 0.7942170625640192,
|
|
"rewards/rejected": -0.31013957346518206,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 19.0971736907959,
|
|
"kl": 7.267646789550781,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35488590.33438486,
|
|
"logits/rejected": -37373460.60681114,
|
|
"logps/chosen": -492.19952681388014,
|
|
"logps/rejected": -354.60083688080493,
|
|
"loss": 0.6182,
|
|
"loss/base_kto": 3.2969772815704346,
|
|
"metrics/advantage_var": 1032.6224365234375,
|
|
"regularization/lipschitz_norm": 1702.734375,
|
|
"regularization/w1": 1.6482470035552979,
|
|
"rewards/chosen": 0.34289632610718157,
|
|
"rewards/margins": 0.7275893183264275,
|
|
"rewards/rejected": -0.38469299221924586,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 20.185989379882812,
|
|
"kl": 13.166139602661133,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34743558.02825746,
|
|
"logits/rejected": -35353362.71228616,
|
|
"logps/chosen": -470.81348116169545,
|
|
"logps/rejected": -391.751676710731,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.2496349811553955,
|
|
"metrics/advantage_var": 596.5450439453125,
|
|
"regularization/lipschitz_norm": 1545.3150634765625,
|
|
"regularization/w1": 1.4958648681640625,
|
|
"rewards/chosen": 0.4876913073672979,
|
|
"rewards/margins": 0.7794211528562778,
|
|
"rewards/rejected": -0.2917298454889799,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 22.58867073059082,
|
|
"kl": 8.902046203613281,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -36490514.202723145,
|
|
"logits/rejected": -36235048.943457186,
|
|
"logps/chosen": -470.6004160363086,
|
|
"logps/rejected": -365.33554624394185,
|
|
"loss": 0.6082,
|
|
"loss/base_kto": 3.3208978176116943,
|
|
"metrics/advantage_var": 587.3114624023438,
|
|
"regularization/lipschitz_norm": 1595.944580078125,
|
|
"regularization/w1": 1.5448744297027588,
|
|
"rewards/chosen": 0.39317979682772314,
|
|
"rewards/margins": 0.7648624029717054,
|
|
"rewards/rejected": -0.37168260614398224,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 25.996488571166992,
|
|
"kl": 16.85188102722168,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -36245958.38985737,
|
|
"logits/rejected": -37731247.53158706,
|
|
"logps/chosen": -478.3878763866878,
|
|
"logps/rejected": -399.99412557781204,
|
|
"loss": 0.6195,
|
|
"loss/base_kto": 3.347888231277466,
|
|
"metrics/advantage_var": 639.7686157226562,
|
|
"regularization/lipschitz_norm": 1661.5614013671875,
|
|
"regularization/w1": 1.6083911657333374,
|
|
"rewards/chosen": 0.4293580939388124,
|
|
"rewards/margins": 0.7200904556250605,
|
|
"rewards/rejected": -0.29073236168624805,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.05373764038086,
|
|
"kl": 13.279568672180176,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -36626845.476340696,
|
|
"logits/rejected": -36377768.0247678,
|
|
"logps/chosen": -473.1984917192429,
|
|
"logps/rejected": -380.3654943885449,
|
|
"loss": 0.5966,
|
|
"loss/base_kto": 3.2414116859436035,
|
|
"metrics/advantage_var": 615.60986328125,
|
|
"regularization/lipschitz_norm": 1581.9222412109375,
|
|
"regularization/w1": 1.5313007831573486,
|
|
"rewards/chosen": 0.5020763505520505,
|
|
"rewards/margins": 0.8529431348043,
|
|
"rewards/rejected": -0.35086678425224943,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 20.508298873901367,
|
|
"kl": 13.969990730285645,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34712249.461300306,
|
|
"logits/rejected": -36584948.69400631,
|
|
"logps/chosen": -505.95714009287923,
|
|
"logps/rejected": -360.8237874605678,
|
|
"loss": 0.6011,
|
|
"loss/base_kto": 3.31241774559021,
|
|
"metrics/advantage_var": 579.796875,
|
|
"regularization/lipschitz_norm": 1545.9078369140625,
|
|
"regularization/w1": 1.4964386224746704,
|
|
"rewards/chosen": 0.4728007124673471,
|
|
"rewards/margins": 0.725570110479719,
|
|
"rewards/rejected": -0.25276939801237186,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 17.613689422607422,
|
|
"kl": 9.625328063964844,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -36149144.935064934,
|
|
"logits/rejected": -36455402.409638554,
|
|
"logps/chosen": -491.9526684253247,
|
|
"logps/rejected": -394.0533932605422,
|
|
"loss": 0.6035,
|
|
"loss/base_kto": 3.3030622005462646,
|
|
"metrics/advantage_var": 564.5618286132812,
|
|
"regularization/lipschitz_norm": 1575.0570068359375,
|
|
"regularization/w1": 1.5246552228927612,
|
|
"rewards/chosen": 0.3459713675759055,
|
|
"rewards/margins": 0.7715479108093863,
|
|
"rewards/rejected": -0.4255765432334808,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 22.337461471557617,
|
|
"kl": 4.979233264923096,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -33656968.75570033,
|
|
"logits/rejected": -35247260.82882883,
|
|
"logps/chosen": -499.64057410423453,
|
|
"logps/rejected": -364.9675065690691,
|
|
"loss": 0.6284,
|
|
"loss/base_kto": 3.332446813583374,
|
|
"metrics/advantage_var": 701.9683837890625,
|
|
"regularization/lipschitz_norm": 1750.4090576171875,
|
|
"regularization/w1": 1.694395899772644,
|
|
"rewards/chosen": 0.24563369999491041,
|
|
"rewards/margins": 0.7634950061252594,
|
|
"rewards/rejected": -0.5178613061303491,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 16.40119743347168,
|
|
"kl": 6.889324188232422,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35966037.19937205,
|
|
"logits/rejected": -36749996.790046655,
|
|
"logps/chosen": -473.3528257456829,
|
|
"logps/rejected": -385.8450136080871,
|
|
"loss": 0.6045,
|
|
"loss/base_kto": 3.298159122467041,
|
|
"metrics/advantage_var": 655.0313720703125,
|
|
"regularization/lipschitz_norm": 1588.679931640625,
|
|
"regularization/w1": 1.537842035293579,
|
|
"rewards/chosen": 0.2532909159757653,
|
|
"rewards/margins": 0.8061385334973925,
|
|
"rewards/rejected": -0.5528476175216271,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 53.743717193603516,
|
|
"kl": 7.675045967102051,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34990409.02295082,
|
|
"logits/rejected": -35224576.0,
|
|
"logps/chosen": -453.2701844262295,
|
|
"logps/rejected": -359.14731809701493,
|
|
"loss": 0.6092,
|
|
"loss/base_kto": 3.2997264862060547,
|
|
"metrics/advantage_var": 766.1932983398438,
|
|
"regularization/lipschitz_norm": 1626.1533203125,
|
|
"regularization/w1": 1.5741164684295654,
|
|
"rewards/chosen": 0.29436165231173156,
|
|
"rewards/margins": 0.7807156033815776,
|
|
"rewards/rejected": -0.4863539510698461,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 17.668380737304688,
|
|
"kl": 6.745764255523682,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34590484.63283582,
|
|
"logits/rejected": -35847936.83934426,
|
|
"logps/chosen": -480.08120335820894,
|
|
"logps/rejected": -386.7669057377049,
|
|
"loss": 0.6055,
|
|
"loss/base_kto": 3.306565523147583,
|
|
"metrics/advantage_var": 677.1510620117188,
|
|
"regularization/lipschitz_norm": 1588.463134765625,
|
|
"regularization/w1": 1.5376323461532593,
|
|
"rewards/chosen": 0.3046172184730644,
|
|
"rewards/margins": 0.7863921208168143,
|
|
"rewards/rejected": -0.48177490234375,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 15.577917098999023,
|
|
"kl": 14.072929382324219,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35061604.7014218,
|
|
"logits/rejected": -34632338.39876352,
|
|
"logps/chosen": -447.5903436018957,
|
|
"logps/rejected": -353.15091286707883,
|
|
"loss": 0.5939,
|
|
"loss/base_kto": 3.302741289138794,
|
|
"metrics/advantage_var": 546.7880249023438,
|
|
"regularization/lipschitz_norm": 1496.0673828125,
|
|
"regularization/w1": 1.4481931924819946,
|
|
"rewards/chosen": 0.4725491251049072,
|
|
"rewards/margins": 0.7593424285577424,
|
|
"rewards/rejected": -0.2867933034528352,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 22.389183044433594,
|
|
"kl": 4.737483501434326,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -33615264.706624605,
|
|
"logits/rejected": -34156249.164086685,
|
|
"logps/chosen": -465.23176261829656,
|
|
"logps/rejected": -351.9534878095975,
|
|
"loss": 0.6037,
|
|
"loss/base_kto": 3.403430700302124,
|
|
"metrics/advantage_var": 504.26251220703125,
|
|
"regularization/lipschitz_norm": 1473.365478515625,
|
|
"regularization/w1": 1.4262179136276245,
|
|
"rewards/chosen": 0.1754674339896121,
|
|
"rewards/margins": 0.6569555939251529,
|
|
"rewards/rejected": -0.48148815993554084,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 26.31852149963379,
|
|
"kl": 8.447528839111328,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34746091.42122187,
|
|
"logits/rejected": -36011440.632218845,
|
|
"logps/chosen": -483.4614147909968,
|
|
"logps/rejected": -387.8470507218845,
|
|
"loss": 0.6054,
|
|
"loss/base_kto": 3.304145574569702,
|
|
"metrics/advantage_var": 665.6144409179688,
|
|
"regularization/lipschitz_norm": 1590.29248046875,
|
|
"regularization/w1": 1.5394030809402466,
|
|
"rewards/chosen": 0.34717412623561844,
|
|
"rewards/margins": 0.7916845426549288,
|
|
"rewards/rejected": -0.4445104164193104,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 25.04899024963379,
|
|
"kl": 6.436589241027832,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35697680.938345864,
|
|
"logits/rejected": -36233239.31056911,
|
|
"logps/chosen": -497.6437969924812,
|
|
"logps/rejected": -380.7557164634146,
|
|
"loss": 0.6238,
|
|
"loss/base_kto": 3.326345205307007,
|
|
"metrics/advantage_var": 629.7172241210938,
|
|
"regularization/lipschitz_norm": 1719.4332275390625,
|
|
"regularization/w1": 1.6644114255905151,
|
|
"rewards/chosen": 0.3143423438968515,
|
|
"rewards/margins": 0.7660934077942092,
|
|
"rewards/rejected": -0.4517510638973577,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 17.320589065551758,
|
|
"kl": 12.672170639038086,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -36098936.401826486,
|
|
"logits/rejected": -36160234.22150883,
|
|
"logps/chosen": -507.1368911719939,
|
|
"logps/rejected": -376.6101023274478,
|
|
"loss": 0.6141,
|
|
"loss/base_kto": 3.317797899246216,
|
|
"metrics/advantage_var": 668.8302612304688,
|
|
"regularization/lipschitz_norm": 1647.47265625,
|
|
"regularization/w1": 1.5947535037994385,
|
|
"rewards/chosen": 0.562269376293165,
|
|
"rewards/margins": 0.7557992186637509,
|
|
"rewards/rejected": -0.19352984237058588,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 29.0887451171875,
|
|
"kl": 12.94732666015625,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34591852.8,
|
|
"logits/rejected": -35478531.2,
|
|
"logps/chosen": -470.71328125,
|
|
"logps/rejected": -373.483056640625,
|
|
"loss": 0.6001,
|
|
"loss/base_kto": 3.2657830715179443,
|
|
"metrics/advantage_var": 609.7000732421875,
|
|
"regularization/lipschitz_norm": 1585.9810791015625,
|
|
"regularization/w1": 1.5352295637130737,
|
|
"rewards/chosen": 0.4682040214538574,
|
|
"rewards/margins": 0.8068231582641602,
|
|
"rewards/rejected": -0.33861913681030276,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 19.229684829711914,
|
|
"kl": 11.90479564666748,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34786442.37837838,
|
|
"logits/rejected": -36989415.61904762,
|
|
"logps/chosen": -466.53120031796504,
|
|
"logps/rejected": -380.4815908218126,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.2592263221740723,
|
|
"metrics/advantage_var": 603.54345703125,
|
|
"regularization/lipschitz_norm": 1569.5506591796875,
|
|
"regularization/w1": 1.5193250179290771,
|
|
"rewards/chosen": 0.45788515997615264,
|
|
"rewards/margins": 0.8053966507115597,
|
|
"rewards/rejected": -0.34751149073540705,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 16.710540771484375,
|
|
"kl": 10.548983573913574,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36278706.138248846,
|
|
"logits/rejected": -36451847.32591415,
|
|
"logps/chosen": -475.1974846390169,
|
|
"logps/rejected": -389.2097823926868,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.272834062576294,
|
|
"metrics/advantage_var": 576.3281860351562,
|
|
"regularization/lipschitz_norm": 1557.21875,
|
|
"regularization/w1": 1.5073877573013306,
|
|
"rewards/chosen": 0.3840080061998968,
|
|
"rewards/margins": 0.7844563671585116,
|
|
"rewards/rejected": -0.40044836095861486,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 19.202665328979492,
|
|
"kl": 14.051457405090332,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -36245189.4272,
|
|
"logits/rejected": -36364750.75419848,
|
|
"logps/chosen": -459.21635,
|
|
"logps/rejected": -380.07800572519085,
|
|
"loss": 0.6111,
|
|
"loss/base_kto": 3.3619213104248047,
|
|
"metrics/advantage_var": 564.919921875,
|
|
"regularization/lipschitz_norm": 1577.078125,
|
|
"regularization/w1": 1.526611566543579,
|
|
"rewards/chosen": 0.3721796875,
|
|
"rewards/margins": 0.6878680522423664,
|
|
"rewards/rejected": -0.3156883647423664,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 21.276033401489258,
|
|
"kl": 9.260282516479492,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35170786.835443035,
|
|
"logits/rejected": -36420816.59259259,
|
|
"logps/chosen": -484.72522745253167,
|
|
"logps/rejected": -379.61487268518516,
|
|
"loss": 0.6178,
|
|
"loss/base_kto": 3.3297855854034424,
|
|
"metrics/advantage_var": 646.2836303710938,
|
|
"regularization/lipschitz_norm": 1666.2606201171875,
|
|
"regularization/w1": 1.6129401922225952,
|
|
"rewards/chosen": 0.3696428854254228,
|
|
"rewards/margins": 0.7356038660644684,
|
|
"rewards/rejected": -0.36596098063904564,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 15.867319107055664,
|
|
"kl": 9.925912857055664,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36473148.30577223,
|
|
"logits/rejected": -36942676.53208138,
|
|
"logps/chosen": -502.1618077223089,
|
|
"logps/rejected": -382.19160798122067,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.288565158843994,
|
|
"metrics/advantage_var": 596.0578002929688,
|
|
"regularization/lipschitz_norm": 1572.09423828125,
|
|
"regularization/w1": 1.5217872858047485,
|
|
"rewards/chosen": 0.35841133962741323,
|
|
"rewards/margins": 0.7684826753020513,
|
|
"rewards/rejected": -0.4100713356746381,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 20.524717330932617,
|
|
"kl": 13.834006309509277,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35917880.97681607,
|
|
"logits/rejected": -37481894.21800948,
|
|
"logps/chosen": -484.5685857805255,
|
|
"logps/rejected": -369.6553120063191,
|
|
"loss": 0.6028,
|
|
"loss/base_kto": 3.2853474617004395,
|
|
"metrics/advantage_var": 629.42626953125,
|
|
"regularization/lipschitz_norm": 1587.9732666015625,
|
|
"regularization/w1": 1.5371580123901367,
|
|
"rewards/chosen": 0.5182484268589403,
|
|
"rewards/margins": 0.7909771765966724,
|
|
"rewards/rejected": -0.27272874973773203,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 22.96837043762207,
|
|
"kl": 11.15085506439209,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34159137.2879257,
|
|
"logits/rejected": -35866908.26498423,
|
|
"logps/chosen": -478.7232488390093,
|
|
"logps/rejected": -390.56873521293375,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.238645315170288,
|
|
"metrics/advantage_var": 549.8868408203125,
|
|
"regularization/lipschitz_norm": 1556.4947509765625,
|
|
"regularization/w1": 1.506687045097351,
|
|
"rewards/chosen": 0.43505859375,
|
|
"rewards/margins": 0.8077497271709262,
|
|
"rewards/rejected": -0.3726911334209262,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 33.45109939575195,
|
|
"kl": 20.301267623901367,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34947843.084337346,
|
|
"logits/rejected": -35470665.14285714,
|
|
"logps/chosen": -460.02748493975906,
|
|
"logps/rejected": -381.2789924918831,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.307748556137085,
|
|
"metrics/advantage_var": 582.9851684570312,
|
|
"regularization/lipschitz_norm": 1541.517578125,
|
|
"regularization/w1": 1.4921890497207642,
|
|
"rewards/chosen": 0.6274261015007295,
|
|
"rewards/margins": 0.7260270649941758,
|
|
"rewards/rejected": -0.09860096349344626,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 17.269893646240234,
|
|
"kl": 19.092384338378906,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34940828.95475819,
|
|
"logits/rejected": -36077503.69858713,
|
|
"logps/chosen": -486.26301677067084,
|
|
"logps/rejected": -362.81171507064363,
|
|
"loss": 0.5659,
|
|
"loss/base_kto": 3.2226264476776123,
|
|
"metrics/advantage_var": 528.5355224609375,
|
|
"regularization/lipschitz_norm": 1347.966064453125,
|
|
"regularization/w1": 1.3048311471939087,
|
|
"rewards/chosen": 0.6287274293706123,
|
|
"rewards/margins": 0.8119409552594835,
|
|
"rewards/rejected": -0.1832135258888712,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 20.47254753112793,
|
|
"kl": 16.843952178955078,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35502233.6,
|
|
"logits/rejected": -35992384.0,
|
|
"logps/chosen": -468.803173828125,
|
|
"logps/rejected": -395.0420654296875,
|
|
"loss": 0.559,
|
|
"loss/base_kto": 3.207951307296753,
|
|
"metrics/advantage_var": 730.568359375,
|
|
"regularization/lipschitz_norm": 1305.5416259765625,
|
|
"regularization/w1": 1.2637642621994019,
|
|
"rewards/chosen": 0.5241402149200439,
|
|
"rewards/margins": 0.8164856910705567,
|
|
"rewards/rejected": -0.2923454761505127,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 15.052715301513672,
|
|
"kl": 15.110455513000488,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34836823.514377,
|
|
"logits/rejected": -36595636.8440367,
|
|
"logps/chosen": -481.13128993610223,
|
|
"logps/rejected": -388.56880733944956,
|
|
"loss": 0.5545,
|
|
"loss/base_kto": 3.1855056285858154,
|
|
"metrics/advantage_var": 413.5293884277344,
|
|
"regularization/lipschitz_norm": 1292.14306640625,
|
|
"regularization/w1": 1.250794529914856,
|
|
"rewards/chosen": 0.5496380108233077,
|
|
"rewards/margins": 0.833160940443613,
|
|
"rewards/rejected": -0.28352292962030534,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 15.298805236816406,
|
|
"kl": 10.011313438415527,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -35729310.63074485,
|
|
"logits/rejected": -36671866.67488444,
|
|
"logps/chosen": -498.35286251980983,
|
|
"logps/rejected": -386.7250577812018,
|
|
"loss": 0.5537,
|
|
"loss/base_kto": 3.1345555782318115,
|
|
"metrics/advantage_var": 446.0438537597656,
|
|
"regularization/lipschitz_norm": 1338.201171875,
|
|
"regularization/w1": 1.2953788042068481,
|
|
"rewards/chosen": 0.5328630305318938,
|
|
"rewards/margins": 0.9120864940635289,
|
|
"rewards/rejected": -0.3792234635316352,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 18.957292556762695,
|
|
"kl": 11.259045600891113,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35911658.344410874,
|
|
"logits/rejected": -36047046.83495145,
|
|
"logps/chosen": -506.5621223564955,
|
|
"logps/rejected": -363.3722694174757,
|
|
"loss": 0.5688,
|
|
"loss/base_kto": 3.192760467529297,
|
|
"metrics/advantage_var": 449.05841064453125,
|
|
"regularization/lipschitz_norm": 1402.3572998046875,
|
|
"regularization/w1": 1.357481837272644,
|
|
"rewards/chosen": 0.4886871983277474,
|
|
"rewards/margins": 0.8561937996058118,
|
|
"rewards/rejected": -0.36750660127806434,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.722339630126953,
|
|
"kl": 11.965067863464355,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34873755.4992272,
|
|
"logits/rejected": -36359574.041074246,
|
|
"logps/chosen": -458.502414992272,
|
|
"logps/rejected": -394.61275671406,
|
|
"loss": 0.5644,
|
|
"loss/base_kto": 3.1753265857696533,
|
|
"metrics/advantage_var": 486.19488525390625,
|
|
"regularization/lipschitz_norm": 1384.5140380859375,
|
|
"regularization/w1": 1.3402096033096313,
|
|
"rewards/chosen": 0.5226662118431221,
|
|
"rewards/margins": 0.872538915455632,
|
|
"rewards/rejected": -0.3498727036125099,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.472070693969727,
|
|
"kl": 13.372385025024414,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -36414990.76176024,
|
|
"logits/rejected": -37644967.368760064,
|
|
"logps/chosen": -481.43294764795144,
|
|
"logps/rejected": -378.47222222222223,
|
|
"loss": 0.5594,
|
|
"loss/base_kto": 3.1960084438323975,
|
|
"metrics/advantage_var": 450.5712890625,
|
|
"regularization/lipschitz_norm": 1321.7607421875,
|
|
"regularization/w1": 1.2794643640518188,
|
|
"rewards/chosen": 0.5018694910909047,
|
|
"rewards/margins": 0.8513860356479448,
|
|
"rewards/rejected": -0.34951654455704007,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 16.251253128051758,
|
|
"kl": 15.280960083007812,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34850750.77707006,
|
|
"logits/rejected": -36636050.06134969,
|
|
"logps/chosen": -489.2749800955414,
|
|
"logps/rejected": -358.795580904908,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.1880719661712646,
|
|
"metrics/advantage_var": 452.4308776855469,
|
|
"regularization/lipschitz_norm": 1323.8626708984375,
|
|
"regularization/w1": 1.281498908996582,
|
|
"rewards/chosen": 0.5583180713046129,
|
|
"rewards/margins": 0.8283061722575102,
|
|
"rewards/rejected": -0.2699881009528973,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 17.741823196411133,
|
|
"kl": 11.286553382873535,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -35456290.7654321,
|
|
"logits/rejected": -36266194.63291139,
|
|
"logps/chosen": -516.8061824845679,
|
|
"logps/rejected": -386.8253065664557,
|
|
"loss": 0.5709,
|
|
"loss/base_kto": 3.181992769241333,
|
|
"metrics/advantage_var": 481.1476745605469,
|
|
"regularization/lipschitz_norm": 1431.2039794921875,
|
|
"regularization/w1": 1.385405421257019,
|
|
"rewards/chosen": 0.5044757701732494,
|
|
"rewards/margins": 0.8764555198286712,
|
|
"rewards/rejected": -0.3719797496554218,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 13.8507080078125,
|
|
"kl": 13.13298225402832,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35048114.052550234,
|
|
"logits/rejected": -36061536.65718799,
|
|
"logps/chosen": -473.1398763523957,
|
|
"logps/rejected": -373.70240916271723,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.181934356689453,
|
|
"metrics/advantage_var": 437.63555908203125,
|
|
"regularization/lipschitz_norm": 1318.9224853515625,
|
|
"regularization/w1": 1.2767168283462524,
|
|
"rewards/chosen": 0.5100622162384081,
|
|
"rewards/margins": 0.883452088567903,
|
|
"rewards/rejected": -0.37338987232949494,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 20.069026947021484,
|
|
"kl": 14.71772289276123,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35693848.74801902,
|
|
"logits/rejected": -37531032.65331279,
|
|
"logps/chosen": -475.4695423930269,
|
|
"logps/rejected": -369.5900423728813,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.1593399047851562,
|
|
"metrics/advantage_var": 480.9468688964844,
|
|
"regularization/lipschitz_norm": 1371.1214599609375,
|
|
"regularization/w1": 1.3272455930709839,
|
|
"rewards/chosen": 0.544325500585009,
|
|
"rewards/margins": 0.8860338322882302,
|
|
"rewards/rejected": -0.3417083317032213,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 13.912832260131836,
|
|
"kl": 16.383499145507812,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34503213.93800979,
|
|
"logits/rejected": -36323604.341829084,
|
|
"logps/chosen": -471.8323817292007,
|
|
"logps/rejected": -363.04155734632684,
|
|
"loss": 0.5497,
|
|
"loss/base_kto": 3.112600088119507,
|
|
"metrics/advantage_var": 434.81622314453125,
|
|
"regularization/lipschitz_norm": 1327.164306640625,
|
|
"regularization/w1": 1.284695029258728,
|
|
"rewards/chosen": 0.5958772693528243,
|
|
"rewards/margins": 0.8821097100782623,
|
|
"rewards/rejected": -0.2862324407254381,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 18.479116439819336,
|
|
"kl": 13.204370498657227,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35202763.836990595,
|
|
"logits/rejected": -36046519.42679128,
|
|
"logps/chosen": -482.3726489028213,
|
|
"logps/rejected": -381.57410922897196,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.229367971420288,
|
|
"metrics/advantage_var": 436.4747619628906,
|
|
"regularization/lipschitz_norm": 1263.3404541015625,
|
|
"regularization/w1": 1.2229136228561401,
|
|
"rewards/chosen": 0.4786768228656446,
|
|
"rewards/margins": 0.8042457855570022,
|
|
"rewards/rejected": -0.3255689626913576,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 23.762554168701172,
|
|
"kl": 9.439775466918945,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -36640960.49306626,
|
|
"logits/rejected": -37119659.20760697,
|
|
"logps/chosen": -494.4505970724191,
|
|
"logps/rejected": -395.56968106180665,
|
|
"loss": 0.5738,
|
|
"loss/base_kto": 3.202040195465088,
|
|
"metrics/advantage_var": 540.9016723632812,
|
|
"regularization/lipschitz_norm": 1434.072509765625,
|
|
"regularization/w1": 1.3881821632385254,
|
|
"rewards/chosen": 0.4762262893935381,
|
|
"rewards/margins": 0.8585141869055478,
|
|
"rewards/rejected": -0.3822878975120097,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 19.653438568115234,
|
|
"kl": 14.8038330078125,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33910057.394034535,
|
|
"logits/rejected": -35809197.1881804,
|
|
"logps/chosen": -487.5484693877551,
|
|
"logps/rejected": -389.678776244168,
|
|
"loss": 0.5617,
|
|
"loss/base_kto": 3.1988933086395264,
|
|
"metrics/advantage_var": 465.9308776855469,
|
|
"regularization/lipschitz_norm": 1337.7279052734375,
|
|
"regularization/w1": 1.2949204444885254,
|
|
"rewards/chosen": 0.5217546735491071,
|
|
"rewards/margins": 0.8685879456257983,
|
|
"rewards/rejected": -0.3468332720766913,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 15.924047470092773,
|
|
"kl": 12.509346008300781,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34098378.14886732,
|
|
"logits/rejected": -34944290.803625375,
|
|
"logps/chosen": -461.7072208737864,
|
|
"logps/rejected": -375.9104512839879,
|
|
"loss": 0.557,
|
|
"loss/base_kto": 3.2230756282806396,
|
|
"metrics/advantage_var": 400.6311340332031,
|
|
"regularization/lipschitz_norm": 1273.7662353515625,
|
|
"regularization/w1": 1.2330057621002197,
|
|
"rewards/chosen": 0.4995985308897148,
|
|
"rewards/margins": 0.8195010488941639,
|
|
"rewards/rejected": -0.31990251800444913,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 21.630062103271484,
|
|
"kl": 12.168811798095703,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34602046.87719298,
|
|
"logits/rejected": -36197406.9261745,
|
|
"logps/chosen": -456.53549890350877,
|
|
"logps/rejected": -363.779572147651,
|
|
"loss": 0.5549,
|
|
"loss/base_kto": 3.2045249938964844,
|
|
"metrics/advantage_var": 391.22833251953125,
|
|
"regularization/lipschitz_norm": 1275.5250244140625,
|
|
"regularization/w1": 1.2347081899642944,
|
|
"rewards/chosen": 0.49545234546326755,
|
|
"rewards/margins": 0.8461880034304445,
|
|
"rewards/rejected": -0.35073565796717704,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 16.752269744873047,
|
|
"kl": 15.846307754516602,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35429417.35702746,
|
|
"logits/rejected": -35459031.72163389,
|
|
"logps/chosen": -476.7097132471729,
|
|
"logps/rejected": -372.67811554462935,
|
|
"loss": 0.554,
|
|
"loss/base_kto": 3.213613510131836,
|
|
"metrics/advantage_var": 399.04248046875,
|
|
"regularization/lipschitz_norm": 1258.4700927734375,
|
|
"regularization/w1": 1.2181990146636963,
|
|
"rewards/chosen": 0.5092161606894942,
|
|
"rewards/margins": 0.7849455492742025,
|
|
"rewards/rejected": -0.2757293885847083,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 17.763147354125977,
|
|
"kl": 12.694317817687988,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35264203.39726027,
|
|
"logits/rejected": -36435372.99518459,
|
|
"logps/chosen": -481.9713184931507,
|
|
"logps/rejected": -378.91768659711073,
|
|
"loss": 0.5566,
|
|
"loss/base_kto": 3.143538475036621,
|
|
"metrics/advantage_var": 476.536865234375,
|
|
"regularization/lipschitz_norm": 1352.3900146484375,
|
|
"regularization/w1": 1.3091135025024414,
|
|
"rewards/chosen": 0.5592938427511416,
|
|
"rewards/margins": 0.908205722093161,
|
|
"rewards/rejected": -0.34891187934201945,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 17.705524444580078,
|
|
"kl": 14.494816780090332,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -36325520.604269296,
|
|
"logits/rejected": -36993478.77198212,
|
|
"logps/chosen": -479.01647167487687,
|
|
"logps/rejected": -379.9303278688525,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.2110283374786377,
|
|
"metrics/advantage_var": 458.9030456542969,
|
|
"regularization/lipschitz_norm": 1331.2030029296875,
|
|
"regularization/w1": 1.288604497909546,
|
|
"rewards/chosen": 0.5020134084917641,
|
|
"rewards/margins": 0.8367455778576826,
|
|
"rewards/rejected": -0.33473216936591843,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 16.476505279541016,
|
|
"kl": 11.64929485321045,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35279416.88888889,
|
|
"logits/rejected": -36192641.62025317,
|
|
"logps/chosen": -503.65933641975306,
|
|
"logps/rejected": -366.5517207278481,
|
|
"loss": 0.5664,
|
|
"loss/base_kto": 3.154219388961792,
|
|
"metrics/advantage_var": 479.1024475097656,
|
|
"regularization/lipschitz_norm": 1422.3133544921875,
|
|
"regularization/w1": 1.3767993450164795,
|
|
"rewards/chosen": 0.5272003456398293,
|
|
"rewards/margins": 0.883051820836378,
|
|
"rewards/rejected": -0.35585147519654864,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 19.683258056640625,
|
|
"kl": 12.130386352539062,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34558758.499224804,
|
|
"logits/rejected": -37057282.82204724,
|
|
"logps/chosen": -482.5875968992248,
|
|
"logps/rejected": -365.2892716535433,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.1678123474121094,
|
|
"metrics/advantage_var": 458.7372131347656,
|
|
"regularization/lipschitz_norm": 1325.9140625,
|
|
"regularization/w1": 1.2834848165512085,
|
|
"rewards/chosen": 0.5414055876029554,
|
|
"rewards/margins": 0.8912255098624289,
|
|
"rewards/rejected": -0.34981992225947345,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 13.819494247436523,
|
|
"kl": 12.839395523071289,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -34219564.24691358,
|
|
"logits/rejected": -35972436.25316456,
|
|
"logps/chosen": -474.5318287037037,
|
|
"logps/rejected": -360.006230221519,
|
|
"loss": 0.548,
|
|
"loss/base_kto": 3.1668202877044678,
|
|
"metrics/advantage_var": 432.35894775390625,
|
|
"regularization/lipschitz_norm": 1257.169677734375,
|
|
"regularization/w1": 1.21694016456604,
|
|
"rewards/chosen": 0.5203971391842689,
|
|
"rewards/margins": 0.8745727550985292,
|
|
"rewards/rejected": -0.3541756159142603,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 18.48135757446289,
|
|
"kl": 12.286941528320312,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35587912.20512821,
|
|
"logits/rejected": -36471233.56097561,
|
|
"logps/chosen": -479.3594751602564,
|
|
"logps/rejected": -382.3421065167683,
|
|
"loss": 0.5609,
|
|
"loss/base_kto": 3.1945881843566895,
|
|
"metrics/advantage_var": 475.1148376464844,
|
|
"regularization/lipschitz_norm": 1335.7503662109375,
|
|
"regularization/w1": 1.2930063009262085,
|
|
"rewards/chosen": 0.4915930919158153,
|
|
"rewards/margins": 0.8560660626457361,
|
|
"rewards/rejected": -0.3644729707299209,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 15.279671669006348,
|
|
"kl": 10.108532905578613,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35131598.08805031,
|
|
"logits/rejected": -36985169.0931677,
|
|
"logps/chosen": -499.40163128930817,
|
|
"logps/rejected": -360.122573757764,
|
|
"loss": 0.5597,
|
|
"loss/base_kto": 3.135068893432617,
|
|
"metrics/advantage_var": 469.2569885253906,
|
|
"regularization/lipschitz_norm": 1387.3138427734375,
|
|
"regularization/w1": 1.3429197072982788,
|
|
"rewards/chosen": 0.5223021117396325,
|
|
"rewards/margins": 0.9241383885344452,
|
|
"rewards/rejected": -0.4018362767948127,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 19.652721405029297,
|
|
"kl": 11.437362670898438,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34648764.38473283,
|
|
"logits/rejected": -34602221.568,
|
|
"logps/chosen": -492.20753816793894,
|
|
"logps/rejected": -365.406,
|
|
"loss": 0.5549,
|
|
"loss/base_kto": 3.157897710800171,
|
|
"metrics/advantage_var": 424.27667236328125,
|
|
"regularization/lipschitz_norm": 1323.5146484375,
|
|
"regularization/w1": 1.281162142753601,
|
|
"rewards/chosen": 0.5088138492962786,
|
|
"rewards/margins": 0.8953936588665912,
|
|
"rewards/rejected": -0.3865798095703125,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 18.966503143310547,
|
|
"kl": 12.286229133605957,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34868447.56030534,
|
|
"logits/rejected": -35625929.9328,
|
|
"logps/chosen": -481.48635496183203,
|
|
"logps/rejected": -361.57425,
|
|
"loss": 0.547,
|
|
"loss/base_kto": 3.1658780574798584,
|
|
"metrics/advantage_var": 388.05230712890625,
|
|
"regularization/lipschitz_norm": 1249.7313232421875,
|
|
"regularization/w1": 1.2097399234771729,
|
|
"rewards/chosen": 0.5137385011629294,
|
|
"rewards/margins": 0.8542985109285544,
|
|
"rewards/rejected": -0.340560009765625,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 15.371367454528809,
|
|
"kl": 13.053885459899902,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35639681.945288755,
|
|
"logits/rejected": -36455019.009646304,
|
|
"logps/chosen": -448.27564589665656,
|
|
"logps/rejected": -357.48352090032154,
|
|
"loss": 0.5607,
|
|
"loss/base_kto": 3.1836795806884766,
|
|
"metrics/advantage_var": 428.6468811035156,
|
|
"regularization/lipschitz_norm": 1345.2010498046875,
|
|
"regularization/w1": 1.302154779434204,
|
|
"rewards/chosen": 0.5233531360568247,
|
|
"rewards/margins": 0.864479360292531,
|
|
"rewards/rejected": -0.3411262242357064,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 19.316118240356445,
|
|
"kl": 12.715338706970215,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -35832121.187406294,
|
|
"logits/rejected": -38171840.10440457,
|
|
"logps/chosen": -478.2545914542729,
|
|
"logps/rejected": -384.58189743066885,
|
|
"loss": 0.5525,
|
|
"loss/base_kto": 3.1467702388763428,
|
|
"metrics/advantage_var": 433.15673828125,
|
|
"regularization/lipschitz_norm": 1314.9112548828125,
|
|
"regularization/w1": 1.272834062576294,
|
|
"rewards/chosen": 0.5485547936480978,
|
|
"rewards/margins": 0.8865040143894569,
|
|
"rewards/rejected": -0.3379492207413591,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 22.878684997558594,
|
|
"kl": 15.296401023864746,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35818787.03157895,
|
|
"logits/rejected": -35980153.13170732,
|
|
"logps/chosen": -449.38176691729325,
|
|
"logps/rejected": -393.5766768292683,
|
|
"loss": 0.567,
|
|
"loss/base_kto": 3.239769697189331,
|
|
"metrics/advantage_var": 456.552001953125,
|
|
"regularization/lipschitz_norm": 1339.2767333984375,
|
|
"regularization/w1": 1.2964198589324951,
|
|
"rewards/chosen": 0.5090937334792058,
|
|
"rewards/margins": 0.7848669447025289,
|
|
"rewards/rejected": -0.27577321122332316,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 15.534296989440918,
|
|
"kl": 15.45969295501709,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34817069.08176101,
|
|
"logits/rejected": -35436003.37888199,
|
|
"logps/chosen": -488.48525943396226,
|
|
"logps/rejected": -389.73500582298135,
|
|
"loss": 0.5572,
|
|
"loss/base_kto": 3.189300298690796,
|
|
"metrics/advantage_var": 438.80126953125,
|
|
"regularization/lipschitz_norm": 1309.871337890625,
|
|
"regularization/w1": 1.2679554224014282,
|
|
"rewards/chosen": 0.5602866118808962,
|
|
"rewards/margins": 0.8233315496781108,
|
|
"rewards/rejected": -0.26304493779721466,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 24.697383880615234,
|
|
"kl": 14.72576904296875,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -33582909.2614897,
|
|
"logits/rejected": -34387131.7596302,
|
|
"logps/chosen": -485.731131141046,
|
|
"logps/rejected": -370.3166409861325,
|
|
"loss": 0.5497,
|
|
"loss/base_kto": 3.1540329456329346,
|
|
"metrics/advantage_var": 433.7855529785156,
|
|
"regularization/lipschitz_norm": 1284.4091796875,
|
|
"regularization/w1": 1.243308186531067,
|
|
"rewards/chosen": 0.5517414171608805,
|
|
"rewards/margins": 0.8877764887704481,
|
|
"rewards/rejected": -0.3360350716095676,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 17.93865203857422,
|
|
"kl": 15.8650541305542,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -34308515.93690852,
|
|
"logits/rejected": -36385630.315789476,
|
|
"logps/chosen": -486.77770110410097,
|
|
"logps/rejected": -373.85550503095976,
|
|
"loss": 0.5546,
|
|
"loss/base_kto": 3.1903533935546875,
|
|
"metrics/advantage_var": 419.76568603515625,
|
|
"regularization/lipschitz_norm": 1287.7547607421875,
|
|
"regularization/w1": 1.2465466260910034,
|
|
"rewards/chosen": 0.5353452279364649,
|
|
"rewards/margins": 0.8289449737429906,
|
|
"rewards/rejected": -0.29359974580652576,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 15.090234756469727,
|
|
"kl": 15.1723051071167,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -34608231.975384615,
|
|
"logits/rejected": -36096546.13333333,
|
|
"logps/chosen": -472.8344230769231,
|
|
"logps/rejected": -378.5605654761905,
|
|
"loss": 0.5611,
|
|
"loss/base_kto": 3.192389726638794,
|
|
"metrics/advantage_var": 450.1095275878906,
|
|
"regularization/lipschitz_norm": 1339.0572509765625,
|
|
"regularization/w1": 1.2962074279785156,
|
|
"rewards/chosen": 0.5453052696814904,
|
|
"rewards/margins": 0.8404648911472642,
|
|
"rewards/rejected": -0.2951596214657738,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 26.889219284057617,
|
|
"kl": 17.18345832824707,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -33520245.385365855,
|
|
"logits/rejected": -35923231.9518797,
|
|
"logps/chosen": -485.6486788617886,
|
|
"logps/rejected": -366.3355733082707,
|
|
"loss": 0.552,
|
|
"loss/base_kto": 3.1291427612304688,
|
|
"metrics/advantage_var": 468.8505554199219,
|
|
"regularization/lipschitz_norm": 1329.364013671875,
|
|
"regularization/w1": 1.286824345588684,
|
|
"rewards/chosen": 0.596533203125,
|
|
"rewards/margins": 0.9183046039782072,
|
|
"rewards/rejected": -0.3217714008532072,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 19.001909255981445,
|
|
"kl": 15.974173545837402,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35467042.5945946,
|
|
"logits/rejected": -37187804.21505377,
|
|
"logps/chosen": -464.78596979332275,
|
|
"logps/rejected": -378.59002976190476,
|
|
"loss": 0.556,
|
|
"loss/base_kto": 3.1855247020721436,
|
|
"metrics/advantage_var": 425.10040283203125,
|
|
"regularization/lipschitz_norm": 1304.3060302734375,
|
|
"regularization/w1": 1.2625681161880493,
|
|
"rewards/chosen": 0.5257928360058128,
|
|
"rewards/margins": 0.8431898994178713,
|
|
"rewards/rejected": -0.31739706341205837,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 19.3326416015625,
|
|
"kl": 14.325095176696777,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -36440563.82884311,
|
|
"logits/rejected": -36977911.7164869,
|
|
"logps/chosen": -508.92561410459587,
|
|
"logps/rejected": -351.82148016178735,
|
|
"loss": 0.5573,
|
|
"loss/base_kto": 3.1494834423065186,
|
|
"metrics/advantage_var": 433.21075439453125,
|
|
"regularization/lipschitz_norm": 1352.1075439453125,
|
|
"regularization/w1": 1.3088401556015015,
|
|
"rewards/chosen": 0.5688836389411648,
|
|
"rewards/margins": 0.8805311216970874,
|
|
"rewards/rejected": -0.3116474827559226,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 14.991193771362305,
|
|
"kl": 16.00214195251465,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -36406076.259083726,
|
|
"logits/rejected": -36690094.095826894,
|
|
"logps/chosen": -488.1978672985782,
|
|
"logps/rejected": -390.76463485316845,
|
|
"loss": 0.5473,
|
|
"loss/base_kto": 3.133539915084839,
|
|
"metrics/advantage_var": 452.78546142578125,
|
|
"regularization/lipschitz_norm": 1286.3963623046875,
|
|
"regularization/w1": 1.2452316284179688,
|
|
"rewards/chosen": 0.5499944460900474,
|
|
"rewards/margins": 0.8961078839372025,
|
|
"rewards/rejected": -0.34611343784715515,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_loss": 0.5860469736583492,
|
|
"train_runtime": 11044.9051,
|
|
"train_samples_per_second": 13.42,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978126922009805e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|